TEP_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from glob import glob\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from tqdm import tnrange, tqdm_notebook\n",
    "\n",
    "import ruptures as rpt\n",
    "from utils.custom_cost import CostNew"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 Data importing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TEP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = sorted(glob('TEP_data/*_te.dat'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns=[]\n",
    "for i in range(1, 42):\n",
    "    columns.append(\"XMEAS({})\".format(i))\n",
    "for i in range(1, 12):\n",
    "    columns.append(\"XMV({})\".format(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We do not use the first file because it does not contain a defect\n",
    "test = {}\n",
    "for i, j in enumerate(files[1:], start=1):\n",
    "    test[i] = pd.read_table(j, sep=\"\\s+\", names=columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 Testing ensembling functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "signal = StandardScaler().fit_transform(test[1].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Testing Dynp and DynpEnsembling algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dynp algorithm:\n",
      "Detection delay = 153 min\n"
     ]
    }
   ],
   "source": [
    "print('Dynp algorithm:')\n",
    "algo = rpt.Dynp(custom_cost='l1', jump=1).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=1)\n",
    "print(f'Detection delay = {(my_bkps[0]-160)*3} min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DynpEnsembling algorithm:\n",
      "Detection delay = 153 min\n"
     ]
    }
   ],
   "source": [
    "print('DynpEnsembling algorithm:')\n",
    "c = CostNew()\n",
    "algo = rpt.DynpEnsembling(custom_cost=c, jump=1, ensembling=5).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=1)\n",
    "print(f'Detection delay = {(my_bkps[0]-160)*3} min')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Testing Window and WindowEnsembling algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window algorithm:\n",
      "Detection delay = 168 min\n"
     ]
    }
   ],
   "source": [
    "print('Window algorithm:')\n",
    "algo = rpt.Window(custom_cost='ar', jump=1, width=40).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=1)\n",
    "print(f'Detection delay = {(my_bkps[0]-160)*3} min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WindowEnsembling algorithm:\n",
      "Detection delay = 180 min\n"
     ]
    }
   ],
   "source": [
    "print('WindowEnsembling algorithm:')\n",
    "c = CostNew()\n",
    "algo = rpt.WindowEnsembling(custom_cost=c, jump=1, width=40, ensembling=5).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=1)\n",
    "print(f'Detection delay = {(my_bkps[0]-160)*3} min')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Testing BinSeg and BinSegEnsembling algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BinSeg algorithm:\n",
      "Detection delay = 153 min\n"
     ]
    }
   ],
   "source": [
    "print('BinSeg algorithm:')\n",
    "algo = rpt.Binseg(custom_cost='l2', jump=1).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=1)\n",
    "print(f'Detection delay = {(my_bkps[0]-160)*3} min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BinSegEnsembling algorithm:\n",
      "Detection delay = 153 min\n"
     ]
    }
   ],
   "source": [
    "print('BinSegEnsembling algorithm:')\n",
    "c = CostNew()\n",
    "algo = rpt.BinsegEnsembling(custom_cost=c, jump=1, ensembling=5).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=1)\n",
    "print(f'Detection delay = {(my_bkps[0]-160)*3} min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_CPDE = 17"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 Dynp experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Individual algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dynp(cost, data, params):\n",
    "    ttf = []\n",
    "    for i in data.keys():\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(data[i])\n",
    "        algo = rpt.Dynp(model=cost, \n",
    "                        params=params, \n",
    "                        jump=1)\n",
    "        algo.fit(signal)\n",
    "\n",
    "        my_bkps = algo.predict(n_bkps=1)\n",
    "        ttf.append(my_bkps[0]-160)\n",
    "    return pd.DataFrame({(cost+' '+str(*params.values())): ttf}).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = (\n",
    "    {'cost':'ar', 'params':{'order':1}},\n",
    "    {'cost':'mahalanobis', 'params':{}},\n",
    "    {'cost':'l1', 'params':{}},\n",
    "    {'cost':'l2', 'params':{}},\n",
    "    {'cost':'linear', 'params':{}},\n",
    "#     {'model':'rbf', 'params':{}},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d2b7bf3151e4c2aa71c1b70a724685c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "table = []\n",
    "for model in tqdm_notebook(models):\n",
    "    results = dynp(data=test, **model)\n",
    "    table.append(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (pd.concat(table)*3).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row0_col0,#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row3_col0{\n",
       "            background-color:  #d0e2f2;\n",
       "            color:  #000000;\n",
       "        }#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row0_col1,#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row1_col0{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row1_col1{\n",
       "            background-color:  #7fb9da;\n",
       "            color:  #000000;\n",
       "        }#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row2_col0{\n",
       "            background-color:  #4b98ca;\n",
       "            color:  #000000;\n",
       "        }#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row2_col1{\n",
       "            background-color:  #2373b6;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row3_col1{\n",
       "            background-color:  #3c8cc3;\n",
       "            color:  #000000;\n",
       "        }#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row4_col0,#T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row4_col1{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >FDR, %</th>        <th class=\"col_heading level0 col1\" >ADD (detected)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122level0_row0\" class=\"row_heading level0 row0\" >ar 1</th>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row0_col0\" class=\"data row0 col0\" >9.500000</td>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row0_col1\" class=\"data row0 col1\" >46.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122level0_row1\" class=\"row_heading level0 row1\" >mahalanobis </th>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row1_col0\" class=\"data row1 col0\" >28.600000</td>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row1_col1\" class=\"data row1 col1\" >21.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122level0_row2\" class=\"row_heading level0 row2\" >l1 </th>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row2_col0\" class=\"data row2 col0\" >19.000000</td>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row2_col1\" class=\"data row2 col1\" >34.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122level0_row3\" class=\"row_heading level0 row3\" >l2 </th>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row3_col0\" class=\"data row3 col0\" >9.500000</td>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row3_col1\" class=\"data row3 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122level0_row4\" class=\"row_heading level0 row4\" >linear </th>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row4_col0\" class=\"data row4 col0\" >4.800000</td>\n",
       "                        <td id=\"T_c0c8a66e_8c1a_11eb_ab1a_acde48001122row4_col1\" class=\"data row4 col1\" >0.000000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x111f239a0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_table = (pd.concat(table)*3)\n",
    "\n",
    "pd.DataFrame({'FDR, %':(final_table[(final_table>=0) & (final_table<96)].count(axis=1) / 21 * 100).round(1),\n",
    "              'ADD (detected)':final_table[(final_table>=0) & (final_table<96)].mean(axis=1).round(1)}).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dynpEnsemble(cost, data, num_agg_func):\n",
    "    ttf = []\n",
    "    for i in data.keys():\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(data[i].values)\n",
    "        algo = rpt.DynpEnsembling(custom_cost=cost, \n",
    "                                  jump=1, \n",
    "                                  ensembling=num_agg_func)\n",
    "        algo.fit(signal)\n",
    "\n",
    "        try:\n",
    "            my_bkps = algo.predict(n_bkps=1)\n",
    "            ttf.append(my_bkps[0]-160)\n",
    "        except:\n",
    "            ttf.append(np.NaN)\n",
    "    return pd.DataFrame({(num_agg_func): ttf}).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "840bee83c9a64190b7f93ed8efb3cb28",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "agg functions loop:   0%|          | 0/16 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 22min 54s, sys: 14min 7s, total: 1h 37min 2s\n",
      "Wall time: 16min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cost = CostNew()\n",
    "table1 = []\n",
    "\n",
    "for n in tnrange(1, NUM_CPDE, desc='agg functions loop'):\n",
    "    table1.append(dynpEnsemble(cost=cost, data=test, num_agg_func=n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (pd.concat(table1)*3).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row0_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row3_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row7_col1{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row0_col1,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row3_col1{\n",
       "            background-color:  #84bcdb;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row1_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row4_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row5_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row8_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row9_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row12_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row13_col0{\n",
       "            background-color:  #abd0e6;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row1_col1{\n",
       "            background-color:  #57a0ce;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row2_col0{\n",
       "            background-color:  #d6e6f4;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row2_col1,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row14_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row15_col0{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row4_col1,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row5_col1,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row8_col1,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row9_col1,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row12_col1,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row13_col1{\n",
       "            background-color:  #4e9acb;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row6_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row10_col0{\n",
       "            background-color:  #3787c0;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row6_col1,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row10_col1{\n",
       "            background-color:  #5ca4d0;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row7_col0,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row11_col0{\n",
       "            background-color:  #6aaed6;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row11_col1{\n",
       "            background-color:  #2777b8;\n",
       "            color:  #000000;\n",
       "        }#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row14_col1,#T_0a94a28c_8c1d_11eb_ab1a_acde48001122row15_col1{\n",
       "            background-color:  #000000;\n",
       "            color:  #f1f1f1;\n",
       "        }</style><table id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >FDR, %</th>        <th class=\"col_heading level0 col1\" >ADD (detected)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row0\" class=\"row_heading level0 row0\" >1</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row0_col0\" class=\"data row0 col0\" >28.600000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row0_col1\" class=\"data row0 col1\" >22.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row1\" class=\"row_heading level0 row1\" >2</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row1_col0\" class=\"data row1 col0\" >9.500000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row1_col1\" class=\"data row1 col1\" >28.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row2\" class=\"row_heading level0 row2\" >3</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row2_col0\" class=\"data row2 col0\" >4.800000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row3\" class=\"row_heading level0 row3\" >4</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row3_col0\" class=\"data row3 col0\" >28.600000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row3_col1\" class=\"data row3 col1\" >22.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row4\" class=\"row_heading level0 row4\" >5</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row4_col0\" class=\"data row4 col0\" >9.500000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row4_col1\" class=\"data row4 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row5\" class=\"row_heading level0 row5\" >6</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row5_col0\" class=\"data row5 col0\" >9.500000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row5_col1\" class=\"data row5 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row6\" class=\"row_heading level0 row6\" >7</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row6_col0\" class=\"data row6 col0\" >19.000000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row6_col1\" class=\"data row6 col1\" >27.800000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row7\" class=\"row_heading level0 row7\" >8</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row7_col0\" class=\"data row7 col0\" >14.300000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row7_col1\" class=\"data row7 col1\" >51.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row8\" class=\"row_heading level0 row8\" >9</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row8_col0\" class=\"data row8 col0\" >9.500000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row8_col1\" class=\"data row8 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row9\" class=\"row_heading level0 row9\" >10</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row9_col0\" class=\"data row9 col0\" >9.500000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row9_col1\" class=\"data row9 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row10\" class=\"row_heading level0 row10\" >11</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row10_col0\" class=\"data row10 col0\" >19.000000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row10_col1\" class=\"data row10 col1\" >27.800000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row11\" class=\"row_heading level0 row11\" >12</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row11_col0\" class=\"data row11 col0\" >14.300000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row11_col1\" class=\"data row11 col1\" >37.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row12\" class=\"row_heading level0 row12\" >13</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row12_col0\" class=\"data row12 col0\" >9.500000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row12_col1\" class=\"data row12 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row13\" class=\"row_heading level0 row13\" >14</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row13_col0\" class=\"data row13 col0\" >9.500000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row13_col1\" class=\"data row13 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row14\" class=\"row_heading level0 row14\" >15</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row14_col0\" class=\"data row14 col0\" >0.000000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row14_col1\" class=\"data row14 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122level0_row15\" class=\"row_heading level0 row15\" >16</th>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row15_col0\" class=\"data row15 col0\" >0.000000</td>\n",
       "                        <td id=\"T_0a94a28c_8c1d_11eb_ab1a_acde48001122row15_col1\" class=\"data row15 col1\" >nan</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x13b86d490>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_table = (pd.concat(table1)*3)\n",
    "\n",
    "pd.DataFrame({'FDR, %':(final_table[(final_table>=0) & (final_table<96)].count(axis=1) / 21 * 100).round(1),\n",
    "              'ADD (detected)':final_table[(final_table>=0) & (final_table<96)].mean(axis=1).round(1)}).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4 Win experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 Individual algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = (\n",
    "    {'cost':'ar', 'params':{'order':1}, 'width':10},\n",
    "    {'cost':'ar', 'params':{'order':1}, 'width':15},\n",
    "    {'cost':'ar', 'params':{'order':5}, 'width':20},\n",
    "    {'cost':'mahalanobis', 'params':{}, 'width':10},\n",
    "    {'cost':'mahalanobis', 'params':{}, 'width':15},\n",
    "    {'cost':'mahalanobis', 'params':{}, 'width':20},\n",
    "    {'cost':'l1', 'params':{}, 'width':10},\n",
    "    {'cost':'l1', 'params':{}, 'width':15},\n",
    "    {'cost':'l1', 'params':{}, 'width':20},\n",
    "    {'cost':'l2', 'params':{}, 'width':10},\n",
    "    {'cost':'l2', 'params':{}, 'width':15},\n",
    "    {'cost':'l2', 'params':{}, 'width':20},\n",
    "#     {'cost':'linear', 'params':{}, 'width':10},\n",
    "#     {'cost':'linear', 'params':{}, 'width':40},\n",
    "#     {'cost':'linear', 'params':{}, 'width':100}\n",
    "#     {'model':'rbf', 'params':{}, 'width':40},\n",
    "#     {'model':'rbf', 'params':{}, 'width':100},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wind(cost, data, params, width):\n",
    "    ttf = []\n",
    "    for i in data.keys():\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(data[i].values)\n",
    "        algo = rpt.Window(model=cost, \n",
    "                          params=params, \n",
    "                          width=width, \n",
    "                          jump=1)\n",
    "        algo.fit(signal)\n",
    "\n",
    "        my_bkps = algo.predict(n_bkps=1)\n",
    "        ttf.append(my_bkps[0]-160)\n",
    "    return pd.DataFrame({(cost+' '+str(*params.values()), width): ttf}).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e872829e24640eab4b0428f97846776",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "agg functions loop:   0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "table = []\n",
    "for model in tqdm_notebook(models, desc='agg functions loop'):\n",
    "    table.append(wind(**model, data=test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (pd.concat(table)*3).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row0_col0,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row1_col0,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row2_col0,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row5_col1,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row6_col0,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row7_col0,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row11_col0{\n",
       "            background-color:  #d6e6f4;\n",
       "            color:  #000000;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row0_col1,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row2_col1,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row6_col1,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row11_col1{\n",
       "            background-color:  #9dcae1;\n",
       "            color:  #000000;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row1_col1,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row8_col0{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row3_col0{\n",
       "            background-color:  #3787c0;\n",
       "            color:  #000000;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row3_col1{\n",
       "            background-color:  #e5eff9;\n",
       "            color:  #000000;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row4_col0,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row7_col1{\n",
       "            background-color:  #6aaed6;\n",
       "            color:  #000000;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row4_col1{\n",
       "            background-color:  #b9d6ea;\n",
       "            color:  #000000;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row5_col0,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row10_col1{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row8_col1{\n",
       "            background-color:  #000000;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row9_col0,#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row10_col0{\n",
       "            background-color:  #abd0e6;\n",
       "            color:  #000000;\n",
       "        }#T_4ab6d244_8cd7_11eb_ab1a_acde48001122row9_col1{\n",
       "            background-color:  #084082;\n",
       "            color:  #f1f1f1;\n",
       "        }</style><table id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122\" ><thead>    <tr>        <th class=\"blank\" ></th>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >FDR, %</th>        <th class=\"col_heading level0 col1\" >ADD (detected)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level0_row0\" class=\"row_heading level0 row0\" rowspan=2>ar 1</th>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row0\" class=\"row_heading level1 row0\" >10</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row0_col0\" class=\"data row0 col0\" >4.800000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row0_col1\" class=\"data row0 col1\" >18.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row1\" class=\"row_heading level1 row1\" >15</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row1_col0\" class=\"data row1 col0\" >4.800000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row1_col1\" class=\"data row1 col1\" >9.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level0_row2\" class=\"row_heading level0 row2\" >ar 5</th>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row2\" class=\"row_heading level1 row2\" >20</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row2_col0\" class=\"data row2 col0\" >4.800000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row2_col1\" class=\"data row2 col1\" >18.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level0_row3\" class=\"row_heading level0 row3\" rowspan=3>mahalanobis </th>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row3\" class=\"row_heading level1 row3\" >10</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row3_col0\" class=\"data row3 col0\" >19.000000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row3_col1\" class=\"data row3 col1\" >11.200000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row4\" class=\"row_heading level1 row4\" >15</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row4_col0\" class=\"data row4 col0\" >14.300000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row4_col1\" class=\"data row4 col1\" >16.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row5\" class=\"row_heading level1 row5\" >20</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row5_col0\" class=\"data row5 col0\" >28.600000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row5_col1\" class=\"data row5 col1\" >13.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level0_row6\" class=\"row_heading level0 row6\" rowspan=3>l1 </th>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row6\" class=\"row_heading level1 row6\" >10</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row6_col0\" class=\"data row6 col0\" >4.800000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row6_col1\" class=\"data row6 col1\" >18.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row7\" class=\"row_heading level1 row7\" >15</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row7_col0\" class=\"data row7 col0\" >4.800000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row7_col1\" class=\"data row7 col1\" >21.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row8\" class=\"row_heading level1 row8\" >20</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row8_col0\" class=\"data row8 col0\" >0.000000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row8_col1\" class=\"data row8 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level0_row9\" class=\"row_heading level0 row9\" rowspan=3>l2 </th>\n",
       "                        <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row9\" class=\"row_heading level1 row9\" >10</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row9_col0\" class=\"data row9 col0\" >9.500000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row9_col1\" class=\"data row9 col1\" >31.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row10\" class=\"row_heading level1 row10\" >15</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row10_col0\" class=\"data row10 col0\" >9.500000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row10_col1\" class=\"data row10 col1\" >33.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122level1_row11\" class=\"row_heading level1 row11\" >20</th>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row11_col0\" class=\"data row11 col0\" >4.800000</td>\n",
       "                        <td id=\"T_4ab6d244_8cd7_11eb_ab1a_acde48001122row11_col1\" class=\"data row11 col1\" >18.000000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x13c79fbb0>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_table = (pd.concat(table)*3)\n",
    "\n",
    "pd.DataFrame({'FDR, %':(final_table[(final_table>=0) & (final_table<96)].count(axis=1) / 21 * 100).round(1),\n",
    "              'ADD (detected)':final_table[(final_table>=0) & (final_table<96)].mean(axis=1).round(1)}).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def windowEnsemble(cost, data, num_agg_func, width):\n",
    "    ttf = []\n",
    "    for i in data.keys():\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(data[i].values)\n",
    "        algo = rpt.WindowEnsembling(custom_cost=cost, \n",
    "                                    jump=1, \n",
    "                                    ensembling=num_agg_func, \n",
    "                                    width=width)\n",
    "        algo.fit(signal)\n",
    "\n",
    "        try:\n",
    "            my_bkps = algo.predict(n_bkps=1)\n",
    "            ttf.append(my_bkps[0]-160)\n",
    "        except:\n",
    "            ttf.append(np.NaN)\n",
    "    return pd.DataFrame({(num_agg_func, width): ttf}).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b15b004383754ccba3c6471475010b74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "agg functions loop:   0%|          | 0/16 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77afe4bd136f49a0959cd79fd8a4cc7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1e5b5af51d24d01b3d4aacfcb432819",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b79b585459b45b594bdf45219c8ed5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22df3aca36e64e41a1637b4347c8957f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aad1abc3b0834d8b869199e839ae25d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9efda5d8b759437e8eea5994f236185e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "052d73d477134dc995ee6402f1e1f362",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c81ba3ff9c64ce188448b68154d5498",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "802493951b5f40c7b328b773fcd46d8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "401afa268f5c4fc3bb68e670e2eb1584",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "183c05d652894315b2494e4a3f159be0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28bb68dd46af4f34b29f9c3241adb930",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3786fa749fa4485a82196962b2b561c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef61d2d169694d4dbb9a617b60ecd2b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f6b7faad4144232985ab1f967aee360",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18a7803f2c2646648b338e2d2c6a2794",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "width loop:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18min 6s, sys: 1min 35s, total: 19min 42s\n",
      "Wall time: 9min 25s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cost = CostNew()\n",
    "table1 = []\n",
    "\n",
    "for n in tnrange(1, NUM_CPDE, desc='agg functions loop'):\n",
    "    for w in tqdm_notebook([10, 15, 20], desc='width loop', leave=False):\n",
    "        table1.append(windowEnsemble(cost=cost, data=test, num_agg_func=n, width=w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (pd.concat(table1)*3).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row0_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row2_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row11_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row16_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row18_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row19_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row22_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row25_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row28_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row31_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row35_col0{\n",
       "            background-color:  #4a98c9;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row0_col1{\n",
       "            background-color:  #f4f9fe;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row1_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row5_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row8_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row10_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row15_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row24_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row27_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row30_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row34_col0{\n",
       "            background-color:  #1764ab;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row1_col1{\n",
       "            background-color:  #77b5d9;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row2_col1{\n",
       "            background-color:  #91c3de;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row3_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row7_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row9_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row12_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row13_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row17_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row20_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row23_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row26_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row29_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row32_col0{\n",
       "            background-color:  #94c4df;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row3_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row16_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row18_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row25_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row28_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row31_col1{\n",
       "            background-color:  #bdd7ec;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row4_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row23_col1{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row4_col1{\n",
       "            background-color:  #2b7bba;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row5_col1{\n",
       "            background-color:  #61a7d2;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row6_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row14_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row21_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row33_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row38_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row41_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row43_col0{\n",
       "            background-color:  #d0e1f2;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row6_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row33_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row36_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row37_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row39_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row40_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row42_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row44_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row45_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row46_col0,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row47_col0{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row7_col1{\n",
       "            background-color:  #4695c8;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row8_col1{\n",
       "            background-color:  #4191c6;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row9_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row12_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row13_col1{\n",
       "            background-color:  #8dc1dd;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row10_col1{\n",
       "            background-color:  #3b8bc2;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row11_col1{\n",
       "            background-color:  #58a1cf;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row14_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row21_col1{\n",
       "            background-color:  #abd0e6;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row15_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row24_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row27_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row30_col1{\n",
       "            background-color:  #cee0f2;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row17_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row26_col1{\n",
       "            background-color:  #e1edf8;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row19_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row20_col1{\n",
       "            background-color:  #6aaed6;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row22_col1{\n",
       "            background-color:  #5fa6d1;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row29_col1{\n",
       "            background-color:  #ecf4fb;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row32_col1{\n",
       "            background-color:  #f2f7fd;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row34_col1{\n",
       "            background-color:  #99c7e0;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row35_col1{\n",
       "            background-color:  #3787c0;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row36_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row37_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row39_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row40_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row42_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row44_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row45_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row46_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row47_col1{\n",
       "            background-color:  #000000;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row38_col1,#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row41_col1{\n",
       "            background-color:  #2979b9;\n",
       "            color:  #000000;\n",
       "        }#T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row43_col1{\n",
       "            background-color:  #105ba4;\n",
       "            color:  #f1f1f1;\n",
       "        }</style><table id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122\" ><thead>    <tr>        <th class=\"blank\" ></th>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >FDR, %</th>        <th class=\"col_heading level0 col1\" >ADD (detected)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row0\" class=\"row_heading level0 row0\" rowspan=3>1</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row0\" class=\"row_heading level1 row0\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row0_col0\" class=\"data row0 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row0_col1\" class=\"data row0 col1\" >1.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row1\" class=\"row_heading level1 row1\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row1_col0\" class=\"data row1 col0\" >19.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row1_col1\" class=\"data row1 col1\" >25.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row2\" class=\"row_heading level1 row2\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row2_col0\" class=\"data row2 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row2_col1\" class=\"data row2 col1\" >22.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row3\" class=\"row_heading level0 row3\" rowspan=3>2</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row3\" class=\"row_heading level1 row3\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row3_col0\" class=\"data row3 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row3_col1\" class=\"data row3 col1\" >15.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row4\" class=\"row_heading level1 row4\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row4_col0\" class=\"data row4 col0\" >23.800000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row4_col1\" class=\"data row4 col1\" >38.400000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row5\" class=\"row_heading level1 row5\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row5_col0\" class=\"data row5 col0\" >19.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row5_col1\" class=\"data row5 col1\" >28.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row6\" class=\"row_heading level0 row6\" rowspan=3>3</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row6\" class=\"row_heading level1 row6\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row6_col0\" class=\"data row6 col0\" >4.800000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row7\" class=\"row_heading level1 row7\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row7_col0\" class=\"data row7 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row7_col1\" class=\"data row7 col1\" >33.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row8\" class=\"row_heading level1 row8\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row8_col0\" class=\"data row8 col0\" >19.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row8_col1\" class=\"data row8 col1\" >33.800000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row9\" class=\"row_heading level0 row9\" rowspan=3>4</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row9\" class=\"row_heading level1 row9\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row9_col0\" class=\"data row9 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row9_col1\" class=\"data row9 col1\" >22.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row10\" class=\"row_heading level1 row10\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row10_col0\" class=\"data row10 col0\" >19.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row10_col1\" class=\"data row10 col1\" >35.200000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row11\" class=\"row_heading level1 row11\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row11_col0\" class=\"data row11 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row11_col1\" class=\"data row11 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row12\" class=\"row_heading level0 row12\" rowspan=3>5</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row12\" class=\"row_heading level1 row12\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row12_col0\" class=\"data row12 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row12_col1\" class=\"data row12 col1\" >22.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row13\" class=\"row_heading level1 row13\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row13_col0\" class=\"data row13 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row13_col1\" class=\"data row13 col1\" >22.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row14\" class=\"row_heading level1 row14\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row14_col0\" class=\"data row14 col0\" >4.800000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row14_col1\" class=\"data row14 col1\" >18.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row15\" class=\"row_heading level0 row15\" rowspan=3>6</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row15\" class=\"row_heading level1 row15\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row15_col0\" class=\"data row15 col0\" >19.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row15_col1\" class=\"data row15 col1\" >11.200000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row16\" class=\"row_heading level1 row16\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row16_col0\" class=\"data row16 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row16_col1\" class=\"data row16 col1\" >15.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row17\" class=\"row_heading level1 row17\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row17_col0\" class=\"data row17 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row17_col1\" class=\"data row17 col1\" >6.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row18\" class=\"row_heading level0 row18\" rowspan=3>7</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row18\" class=\"row_heading level1 row18\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row18_col0\" class=\"data row18 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row18_col1\" class=\"data row18 col1\" >15.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row19\" class=\"row_heading level1 row19\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row19_col0\" class=\"data row19 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row19_col1\" class=\"data row19 col1\" >27.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row20\" class=\"row_heading level1 row20\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row20_col0\" class=\"data row20 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row20_col1\" class=\"data row20 col1\" >27.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row21\" class=\"row_heading level0 row21\" rowspan=3>8</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row21\" class=\"row_heading level1 row21\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row21_col0\" class=\"data row21 col0\" >4.800000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row21_col1\" class=\"data row21 col1\" >18.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row22\" class=\"row_heading level1 row22\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row22_col0\" class=\"data row22 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row22_col1\" class=\"data row22 col1\" >29.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row23\" class=\"row_heading level1 row23\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row23_col0\" class=\"data row23 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row23_col1\" class=\"data row23 col1\" >54.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row24\" class=\"row_heading level0 row24\" rowspan=3>9</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row24\" class=\"row_heading level1 row24\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row24_col0\" class=\"data row24 col0\" >19.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row24_col1\" class=\"data row24 col1\" >11.200000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row25\" class=\"row_heading level1 row25\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row25_col0\" class=\"data row25 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row25_col1\" class=\"data row25 col1\" >15.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row26\" class=\"row_heading level1 row26\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row26_col0\" class=\"data row26 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row26_col1\" class=\"data row26 col1\" >6.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row27\" class=\"row_heading level0 row27\" rowspan=3>10</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row27\" class=\"row_heading level1 row27\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row27_col0\" class=\"data row27 col0\" >19.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row27_col1\" class=\"data row27 col1\" >11.200000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row28\" class=\"row_heading level1 row28\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row28_col0\" class=\"data row28 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row28_col1\" class=\"data row28 col1\" >15.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row29\" class=\"row_heading level1 row29\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row29_col0\" class=\"data row29 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row29_col1\" class=\"data row29 col1\" >3.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row30\" class=\"row_heading level0 row30\" rowspan=3>11</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row30\" class=\"row_heading level1 row30\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row30_col0\" class=\"data row30 col0\" >19.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row30_col1\" class=\"data row30 col1\" >11.200000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row31\" class=\"row_heading level1 row31\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row31_col0\" class=\"data row31 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row31_col1\" class=\"data row31 col1\" >15.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row32\" class=\"row_heading level1 row32\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row32_col0\" class=\"data row32 col0\" >9.500000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row32_col1\" class=\"data row32 col1\" >1.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row33\" class=\"row_heading level0 row33\" rowspan=3>12</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row33\" class=\"row_heading level1 row33\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row33_col0\" class=\"data row33 col0\" >4.800000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row33_col1\" class=\"data row33 col1\" >0.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row34\" class=\"row_heading level1 row34\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row34_col0\" class=\"data row34 col0\" >19.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row34_col1\" class=\"data row34 col1\" >21.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row35\" class=\"row_heading level1 row35\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row35_col0\" class=\"data row35 col0\" >14.300000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row35_col1\" class=\"data row35 col1\" >36.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row36\" class=\"row_heading level0 row36\" rowspan=3>13</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row36\" class=\"row_heading level1 row36\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row36_col0\" class=\"data row36 col0\" >0.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row36_col1\" class=\"data row36 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row37\" class=\"row_heading level1 row37\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row37_col0\" class=\"data row37 col0\" >0.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row37_col1\" class=\"data row37 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row38\" class=\"row_heading level1 row38\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row38_col0\" class=\"data row38 col0\" >4.800000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row38_col1\" class=\"data row38 col1\" >39.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row39\" class=\"row_heading level0 row39\" rowspan=3>14</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row39\" class=\"row_heading level1 row39\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row39_col0\" class=\"data row39 col0\" >0.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row39_col1\" class=\"data row39 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row40\" class=\"row_heading level1 row40\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row40_col0\" class=\"data row40 col0\" >0.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row40_col1\" class=\"data row40 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row41\" class=\"row_heading level1 row41\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row41_col0\" class=\"data row41 col0\" >4.800000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row41_col1\" class=\"data row41 col1\" >39.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row42\" class=\"row_heading level0 row42\" rowspan=3>15</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row42\" class=\"row_heading level1 row42\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row42_col0\" class=\"data row42 col0\" >0.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row42_col1\" class=\"data row42 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row43\" class=\"row_heading level1 row43\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row43_col0\" class=\"data row43 col0\" >4.800000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row43_col1\" class=\"data row43 col1\" >45.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row44\" class=\"row_heading level1 row44\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row44_col0\" class=\"data row44 col0\" >0.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row44_col1\" class=\"data row44 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level0_row45\" class=\"row_heading level0 row45\" rowspan=3>16</th>\n",
       "                        <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row45\" class=\"row_heading level1 row45\" >10</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row45_col0\" class=\"data row45 col0\" >0.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row45_col1\" class=\"data row45 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row46\" class=\"row_heading level1 row46\" >15</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row46_col0\" class=\"data row46 col0\" >0.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row46_col1\" class=\"data row46 col1\" >nan</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122level1_row47\" class=\"row_heading level1 row47\" >20</th>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row47_col0\" class=\"data row47 col0\" >0.000000</td>\n",
       "                        <td id=\"T_9bf32e9a_8cd8_11eb_ab1a_acde48001122row47_col1\" class=\"data row47 col1\" >nan</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x13cbf95b0>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_table = (pd.concat(table1)*3)\n",
    "\n",
    "pd.DataFrame({'FDR, %':(final_table[(final_table>=0) & (final_table<96)].count(axis=1) / 21 * 100).round(1),\n",
    "              'ADD (detected)':final_table[(final_table>=0) & (final_table<96)].mean(axis=1).round(1)}).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5 BinSeg experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.1 Individual algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = (\n",
    "    {'cost':'ar', 'params':{'order':1}},\n",
    "    {'cost':'mahalanobis', 'params':{}},\n",
    "    {'cost':'l1', 'params':{}},\n",
    "    {'cost':'l2', 'params':{}},\n",
    "    {'cost':'linear', 'params':{}},\n",
    "#     {'model':'rbf', 'params':{}},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binseg(cost, data, params):\n",
    "    ttf = []\n",
    "    for i in data.keys():\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(data[i].values)\n",
    "        algo = rpt.Binseg(model=cost, \n",
    "                          params=params, \n",
    "                          jump=1)\n",
    "        algo.fit(signal)\n",
    "\n",
    "        my_bkps = algo.predict(n_bkps=1)\n",
    "        ttf.append(my_bkps[0]-160)\n",
    "    return pd.DataFrame({(cost+' '+str(*params.values())): ttf}).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "793ec000186e4e8caf5fdf4456c39823",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "agg functions loop:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "table = []\n",
    "for model in tqdm_notebook(models, desc='agg functions loop'):\n",
    "    table.append(binseg(**model, data=test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (pd.concat(table)*3).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_7875c94c_8c1e_11eb_ab1a_acde48001122row0_col0,#T_7875c94c_8c1e_11eb_ab1a_acde48001122row3_col0{\n",
       "            background-color:  #d0e2f2;\n",
       "            color:  #000000;\n",
       "        }#T_7875c94c_8c1e_11eb_ab1a_acde48001122row0_col1,#T_7875c94c_8c1e_11eb_ab1a_acde48001122row1_col0{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_7875c94c_8c1e_11eb_ab1a_acde48001122row1_col1{\n",
       "            background-color:  #7fb9da;\n",
       "            color:  #000000;\n",
       "        }#T_7875c94c_8c1e_11eb_ab1a_acde48001122row2_col0{\n",
       "            background-color:  #4b98ca;\n",
       "            color:  #000000;\n",
       "        }#T_7875c94c_8c1e_11eb_ab1a_acde48001122row2_col1{\n",
       "            background-color:  #2373b6;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_7875c94c_8c1e_11eb_ab1a_acde48001122row3_col1{\n",
       "            background-color:  #3c8cc3;\n",
       "            color:  #000000;\n",
       "        }#T_7875c94c_8c1e_11eb_ab1a_acde48001122row4_col0,#T_7875c94c_8c1e_11eb_ab1a_acde48001122row4_col1{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >FDR, %</th>        <th class=\"col_heading level0 col1\" >ADD (detected)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122level0_row0\" class=\"row_heading level0 row0\" >ar 1</th>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row0_col0\" class=\"data row0 col0\" >9.500000</td>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row0_col1\" class=\"data row0 col1\" >46.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122level0_row1\" class=\"row_heading level0 row1\" >mahalanobis </th>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row1_col0\" class=\"data row1 col0\" >28.600000</td>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row1_col1\" class=\"data row1 col1\" >21.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122level0_row2\" class=\"row_heading level0 row2\" >l1 </th>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row2_col0\" class=\"data row2 col0\" >19.000000</td>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row2_col1\" class=\"data row2 col1\" >34.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122level0_row3\" class=\"row_heading level0 row3\" >l2 </th>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row3_col0\" class=\"data row3 col0\" >9.500000</td>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row3_col1\" class=\"data row3 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122level0_row4\" class=\"row_heading level0 row4\" >linear </th>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row4_col0\" class=\"data row4 col0\" >4.800000</td>\n",
       "                        <td id=\"T_7875c94c_8c1e_11eb_ab1a_acde48001122row4_col1\" class=\"data row4 col1\" >0.000000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x13d17b250>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_table = (pd.concat(table)*3)\n",
    "\n",
    "pd.DataFrame({'FDR, %':(final_table[(final_table>=0) & (final_table<96)].count(axis=1) / 21 * 100).round(1),\n",
    "              'ADD (detected)':final_table[(final_table>=0) & (final_table<96)].mean(axis=1).round(1)}).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.2 Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binsegEnsemble(cost, data, num_agg_func):\n",
    "    ttf = []\n",
    "    for i in data.keys():\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(data[i].values)\n",
    "        algo = rpt.BinsegEnsembling(custom_cost=cost, \n",
    "                                    jump=1, \n",
    "                                    ensembling=num_agg_func)\n",
    "        algo.fit(signal)\n",
    "\n",
    "        try:\n",
    "            my_bkps = algo.predict(n_bkps=1)\n",
    "            ttf.append(my_bkps[0]-160)\n",
    "        except:\n",
    "            ttf.append(np.NaN)\n",
    "    return pd.DataFrame({(num_agg_func): ttf}).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8f11cb23dbd4ac09dd4396e8c192567",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "agg functions loop:   0%|          | 0/16 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2h 25min 13s, sys: 22min 18s, total: 2h 47min 32s\n",
      "Wall time: 28min 2s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cost = CostNew()\n",
    "table1 = []\n",
    "\n",
    "for n in tnrange(1, NUM_CPDE, desc='agg functions loop'):\n",
    "    table1.append(binsegEnsemble(cost, data=test, num_agg_func=n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (pd.concat(table1)*3).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_63a74988_8c22_11eb_ab1a_acde48001122row0_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row3_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row7_col1{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_63a74988_8c22_11eb_ab1a_acde48001122row0_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row3_col1{\n",
       "            background-color:  #84bcdb;\n",
       "            color:  #000000;\n",
       "        }#T_63a74988_8c22_11eb_ab1a_acde48001122row1_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row4_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row5_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row6_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row8_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row9_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row10_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row12_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row13_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row14_col0{\n",
       "            background-color:  #abd0e6;\n",
       "            color:  #000000;\n",
       "        }#T_63a74988_8c22_11eb_ab1a_acde48001122row1_col1{\n",
       "            background-color:  #57a0ce;\n",
       "            color:  #000000;\n",
       "        }#T_63a74988_8c22_11eb_ab1a_acde48001122row2_col0{\n",
       "            background-color:  #d6e6f4;\n",
       "            color:  #000000;\n",
       "        }#T_63a74988_8c22_11eb_ab1a_acde48001122row2_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row15_col0{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }#T_63a74988_8c22_11eb_ab1a_acde48001122row4_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row5_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row6_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row8_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row9_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row10_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row12_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row13_col1,#T_63a74988_8c22_11eb_ab1a_acde48001122row14_col1{\n",
       "            background-color:  #4e9acb;\n",
       "            color:  #000000;\n",
       "        }#T_63a74988_8c22_11eb_ab1a_acde48001122row7_col0,#T_63a74988_8c22_11eb_ab1a_acde48001122row11_col0{\n",
       "            background-color:  #6aaed6;\n",
       "            color:  #000000;\n",
       "        }#T_63a74988_8c22_11eb_ab1a_acde48001122row11_col1{\n",
       "            background-color:  #2777b8;\n",
       "            color:  #000000;\n",
       "        }#T_63a74988_8c22_11eb_ab1a_acde48001122row15_col1{\n",
       "            background-color:  #000000;\n",
       "            color:  #f1f1f1;\n",
       "        }</style><table id=\"T_63a74988_8c22_11eb_ab1a_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >FDR, %</th>        <th class=\"col_heading level0 col1\" >ADD (detected)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row0\" class=\"row_heading level0 row0\" >1</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row0_col0\" class=\"data row0 col0\" >28.600000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row0_col1\" class=\"data row0 col1\" >22.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row1\" class=\"row_heading level0 row1\" >2</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row1_col0\" class=\"data row1 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row1_col1\" class=\"data row1 col1\" >28.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row2\" class=\"row_heading level0 row2\" >3</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row2_col0\" class=\"data row2 col0\" >4.800000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row3\" class=\"row_heading level0 row3\" >4</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row3_col0\" class=\"data row3 col0\" >28.600000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row3_col1\" class=\"data row3 col1\" >22.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row4\" class=\"row_heading level0 row4\" >5</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row4_col0\" class=\"data row4 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row4_col1\" class=\"data row4 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row5\" class=\"row_heading level0 row5\" >6</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row5_col0\" class=\"data row5 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row5_col1\" class=\"data row5 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row6\" class=\"row_heading level0 row6\" >7</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row6_col0\" class=\"data row6 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row6_col1\" class=\"data row6 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row7\" class=\"row_heading level0 row7\" >8</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row7_col0\" class=\"data row7 col0\" >14.300000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row7_col1\" class=\"data row7 col1\" >51.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row8\" class=\"row_heading level0 row8\" >9</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row8_col0\" class=\"data row8 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row8_col1\" class=\"data row8 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row9\" class=\"row_heading level0 row9\" >10</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row9_col0\" class=\"data row9 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row9_col1\" class=\"data row9 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row10\" class=\"row_heading level0 row10\" >11</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row10_col0\" class=\"data row10 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row10_col1\" class=\"data row10 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row11\" class=\"row_heading level0 row11\" >12</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row11_col0\" class=\"data row11 col0\" >14.300000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row11_col1\" class=\"data row11 col1\" >37.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row12\" class=\"row_heading level0 row12\" >13</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row12_col0\" class=\"data row12 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row12_col1\" class=\"data row12 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row13\" class=\"row_heading level0 row13\" >14</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row13_col0\" class=\"data row13 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row13_col1\" class=\"data row13 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row14\" class=\"row_heading level0 row14\" >15</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row14_col0\" class=\"data row14 col0\" >9.500000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row14_col1\" class=\"data row14 col1\" >30.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_63a74988_8c22_11eb_ab1a_acde48001122level0_row15\" class=\"row_heading level0 row15\" >16</th>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row15_col0\" class=\"data row15 col0\" >0.000000</td>\n",
       "                        <td id=\"T_63a74988_8c22_11eb_ab1a_acde48001122row15_col1\" class=\"data row15 col1\" >nan</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x13c51d850>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_table = (pd.concat(table1)*3)\n",
    "\n",
    "pd.DataFrame({'FDR, %':(final_table[(final_table>=0) & (final_table<96)].count(axis=1) / 21 * 100).round(1),\n",
    "              'ADD (detected)':final_table[(final_table>=0) & (final_table<96)].mean(axis=1).round(1)}).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}