Deep_Feature_Selection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Deep-Feature-Selection.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "-EC8S0KbRYch",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt install tree\n",
        "!pip install pytorch-lightning\n",
        "# !pip install tensorboardX"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Sc5L36tDX4T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "llkg35VxTDUt",
        "colab_type": "text"
      },
      "source": [
        "## Data loading "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UA9apQQAKvf5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "df = pd.read_csv('/gdrive/My Drive/DL test/train.csv')\n",
        "\n",
        "df = df.astype('float32')\n",
        "\n",
        "y = df['y']\n",
        "df.drop(columns=['y'], inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6urswjldp0bQ",
        "colab_type": "text"
      },
      "source": [
        "We have deal with unbalanced dataset:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gwmmCFqCqBF1",
        "colab_type": "code",
        "outputId": "e8e6f452-6b69-4931-c213-d42f2df73a91",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        }
      },
      "source": [
        "y.value_counts().plot(kind='bar');"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAECCAYAAADw0Rw8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAELlJREFUeJzt3X+s3XV9x/Hna1RQcaH8uGuwLSsZ\nnYYtAfGO1LgsG50bRWP5QxhmkYY0qclwk7lkdvuHf/wDkmUoycbWWLdiHMg6DR0SN1IwZtlALz+C\nAjquCLYN0CsCToki+t4f99NxuWu55/Se2yOf+3wkJ+fzfX8/3/N936R93W8+93vOSVUhSerXL4y7\nAUnS0jLoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ1bMe4GAE477bRat27duNuQ\npNeUe++997tVNbHQvJ+LoF+3bh1TU1PjbkOSXlOSPDHIPJduJKlzBr0kdW6goE/yp0keSvL1JDcl\neX2SM5Pck2Q6yWeTHN/mntC2p9v+dUv5A0iSXt2CQZ9kNfAnwGRV/TpwHHAZcC1wXVWdBTwLbG2H\nbAWebfXr2jxJ0pgMunSzAnhDkhXAG4EngQuA3W3/LuDiNt7ctmn7NybJaNqVJA1rwaCvqgPAXwHf\nYTbgnwfuBZ6rqpfatP3A6jZeDexrx77U5p862rYlSYMaZOnmZGav0s8E3gycCFy42BMn2ZZkKsnU\nzMzMYl9OknQEgyzd/C7w7aqaqaqfAJ8D3gmsbEs5AGuAA218AFgL0PafBDwz/0WrakdVTVbV5MTE\ngvf7S5KO0iBB/x1gQ5I3trX2jcDDwF3A+9qcLcCtbbynbdP231l+Ma0kjc2C74ytqnuS7AbuA14C\n7gd2AF8Abk7ysVbb2Q7ZCXw6yTTwPWbv0OnCuu1fGHcLXXn8mnePuwVpWRjoIxCq6mrg6nnlx4Dz\nDzP3R8Ali29NkjQKvjNWkjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BL\nUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1LnFgz6JG9J8sCcx/eTXJXklCR3\nJHm0PZ/c5ifJ9UmmkzyY5Lyl/zEkSUeyYNBX1Ter6tyqOhd4O/AC8HlgO7C3qtYDe9s2wCZgfXts\nA25YisYlSYMZdulmI/CtqnoC2AzsavVdwMVtvBm4sWbdDaxMcvpIupUkDW3YoL8MuKmNV1XVk238\nFLCqjVcD++Ycs7/VXiHJtiRTSaZmZmaGbEOSNKiBgz7J8cB7gX+ev6+qCqhhTlxVO6pqsqomJyYm\nhjlUkjSEYa7oNwH3VdXTbfvpQ0sy7flgqx8A1s45bk2rSZLGYJigfz8vL9sA7AG2tPEW4NY59cvb\n3TcbgOfnLPFIko6xFYNMSnIi8C7gg3PK1wC3JNkKPAFc2uq3AxcB08zeoXPFyLqVJA1toKCvqh8C\np86rPcPsXTjz5xZw5Ui6kyQtmu+MlaTOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXO\noJekzhn0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdM+glqXMGvSR1zqCXpM4NFPRJVibZneQbSR5J8o4k\npyS5I8mj7fnkNjdJrk8yneTBJOct7Y8gSXo1g17RfwL4YlW9FTgHeATYDuytqvXA3rYNsAlY3x7b\ngBtG2rEkaSgLBn2Sk4DfAnYCVNWLVfUcsBnY1abtAi5u483AjTXrbmBlktNH3rkkaSCDXNGfCcwA\n/5Dk/iSfTHIisKqqnmxzngJWtfFqYN+c4/e3miRpDAYJ+hXAecANVfU24Ie8vEwDQFUVUMOcOMm2\nJFNJpmZmZoY5VJI0hEGCfj+wv6ruadu7mQ3+pw8tybTng23/AWDtnOPXtNorVNWOqpqsqsmJiYmj\n7V+StIAFg76qngL2JXlLK20EHgb2AFtabQtwaxvvAS5vd99sAJ6fs8QjSTrGVgw474+BzyQ5HngM\nuILZXxK3JNkKPAFc2ubeDlwETAMvtLmSpDEZKOir6gFg8jC7Nh5mbgFXLrIvSdKI+M5YSeqcQS9J\nnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5\ng16SOmfQS1LnDHpJ6txAQZ/k8SRfS/JAkqlWOyXJHUkebc8nt3qSXJ9kOsmDSc5byh9AkvTqhrmi\n/52qOreqDn2l4HZgb1WtB/a2bYBNwPr22AbcMKpmJUnDW8zSzWZgVxvvAi6eU7+xZt0NrExy+iLO\nI0lahEGDvoB/T3Jvkm2ttqqqnmzjp4BVbbwa2Dfn2P2tJkkagxUDzvvNqjqQ5JeAO5J8Y+7Oqqok\nNcyJ2y+MbQBnnHHGMIdKkoYw0BV9VR1ozweBzwPnA08fWpJpzwfb9APA2jmHr2m1+a+5o6omq2py\nYmLi6H8CSdKrWjDok5yY5BcPjYHfA74O7AG2tGlbgFvbeA9webv7ZgPw/JwlHknSMTbI0s0q4PNJ\nDs3/p6r6YpKvArck2Qo8AVza5t8OXARMAy8AV4y8a0nSwBYM+qp6DDjnMPVngI2HqRdw5Ui6kyQt\nmu+MlaTOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1Ln\nDHpJ6pxBL0mdM+glqXMGvSR1zqCXpM4NHPRJjktyf5Lb2vaZSe5JMp3ks0mOb/UT2vZ0279uaVqX\nJA1imCv6DwOPzNm+Friuqs4CngW2tvpW4NlWv67NkySNyUBBn2QN8G7gk207wAXA7jZlF3BxG29u\n27T9G9t8SdIYDHpF/3Hgz4Gfte1Tgeeq6qW2vR9Y3cargX0Abf/zbb4kaQwWDPok7wEOVtW9ozxx\nkm1JppJMzczMjPKlJUlzDHJF/07gvUkeB25mdsnmE8DKJCvanDXAgTY+AKwFaPtPAp6Z/6JVtaOq\nJqtqcmJiYlE/hCTpyBYM+qr6i6paU1XrgMuAO6vqD4G7gPe1aVuAW9t4T9um7b+zqmqkXUuSBraY\n++g/CnwkyTSza/A7W30ncGqrfwTYvrgWJUmLsWLhKS+rqi8BX2rjx4DzDzPnR8AlI+hNkjQCvjNW\nkjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWp\ncwa9JHXOoJekzhn0ktQ5g16SOrfgN0wleT3wZeCENn93VV2d5Exmvyz8VOBe4ANV9WKSE4Abgbcz\n+6Xgf1BVjy9R/5KAddu/MO4WuvL4Ne8edwsjNcgV/Y+BC6rqHOBc4MIkG4Brgeuq6izgWWBrm78V\neLbVr2vzJEljsmDQ16wftM3XtUcBFwC7W30XcHEbb27btP0bk2RkHUuShjLQGn2S45I8ABwE7gC+\nBTxXVS+1KfuB1W28GtgH0PY/z+zyjiRpDAYK+qr6aVWdC6wBzgfeutgTJ9mWZCrJ1MzMzGJfTpJ0\nBEPddVNVzwF3Ae8AViY59MfcNcCBNj4ArAVo+09i9o+y819rR1VNVtXkxMTEUbYvSVrIgkGfZCLJ\nyjZ+A/Au4BFmA/99bdoW4NY23tO2afvvrKoaZdOSpMEteHslcDqwK8lxzP5iuKWqbkvyMHBzko8B\n9wM72/ydwKeTTAPfAy5bgr4lSQNaMOir6kHgbYepP8bsev38+o+AS0bSnSRp0XxnrCR1zqCXpM4Z\n9JLUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEv\nSZ0z6CWpcwa9JHXOoJekzg3y5eBrk9yV5OEkDyX5cKufkuSOJI+255NbPUmuTzKd5MEk5y31DyFJ\nOrJBruhfAv6sqs4GNgBXJjkb2A7srar1wN62DbAJWN8e24AbRt61JGlgCwZ9VT1ZVfe18f8AjwCr\ngc3ArjZtF3BxG28GbqxZdwMrk5w+8s4lSQMZao0+yTrgbcA9wKqqerLtegpY1cargX1zDtvfavNf\na1uSqSRTMzMzQ7YtSRrUwEGf5E3AvwBXVdX35+6rqgJqmBNX1Y6qmqyqyYmJiWEOlSQNYaCgT/I6\nZkP+M1X1uVZ++tCSTHs+2OoHgLVzDl/TapKkMRjkrpsAO4FHquqv5+zaA2xp4y3ArXPql7e7bzYA\nz89Z4pEkHWMrBpjzTuADwNeSPNBqfwlcA9ySZCvwBHBp23c7cBEwDbwAXDHSjiVJQ1kw6KvqP4Ac\nYffGw8wv4MpF9iVJGhHfGStJnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z\n6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdG+Q7Yz+V5GCSr8+pnZLkjiSPtueT\nWz1Jrk8yneTBJOctZfOSpIUNckX/j8CF82rbgb1VtR7Y27YBNgHr22MbcMNo2pQkHa0Fg76qvgx8\nb155M7CrjXcBF8+p31iz7gZWJjl9VM1KkoZ3tGv0q6rqyTZ+CljVxquBfXPm7W81SdKYLPqPsVVV\nQA17XJJtSaaSTM3MzCy2DUnSERxt0D99aEmmPR9s9QPA2jnz1rTa/1NVO6pqsqomJyYmjrINSdJC\njjbo9wBb2ngLcOuc+uXt7psNwPNzlngkSWOwYqEJSW4Cfhs4Lcl+4GrgGuCWJFuBJ4BL2/TbgYuA\naeAF4Iol6FmSNIQFg76q3n+EXRsPM7eAKxfblCRpdHxnrCR1zqCXpM4Z9JLUOYNekjpn0EtS5wx6\nSeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHVuSYI+\nyYVJvplkOsn2pTiHJGkwIw/6JMcBfwNsAs4G3p/k7FGfR5I0mKW4oj8fmK6qx6rqReBmYPMSnEeS\nNIClCPrVwL452/tbTZI0BivGdeIk24BtbfMHSb45rl46dBrw3XE3sZBcO+4ONAb+2xytXx5k0lIE\n/QFg7ZztNa32ClW1A9ixBOdf9pJMVdXkuPuQ5vPf5ngsxdLNV4H1Sc5McjxwGbBnCc4jSRrAyK/o\nq+qlJB8C/g04DvhUVT006vNIkgazJGv0VXU7cPtSvLYG4pKYfl75b3MMUlXj7kGStIT8CARJ6pxB\nL0mdM+glqXMGvaQll+SUJKeMu4/lyqDvRJJVSc5rj1Xj7kdKckaSm5PMAPcAX0lysNXWjbe75cW7\nbl7jkpwL/B1wEi+/A3kN8BzwR1V137h60/KW5L+AjwO7q+qnrXYccAlwVVVtGGd/y4lB/xqX5AHg\ng1V1z7z6BuDvq+qc8XSm5S7Jo1W1fth9Gr2xfaiZRubE+SEPUFV3JzlxHA1Jzb1J/hbYxcufaLsW\n2ALcP7auliGv6F/jklwP/ApwI6/8z3Q58O2q+tC4etPy1j7raiuz30dx6KPK9wP/Cuysqh+Pq7fl\nxqDvQJJNvPI/0wFgT/soCknLnEEv6ZhL8p6qum3cfSwX3l7ZsfblLtLPo98YdwPLiX+M7VvG3YCW\ntyRv5fDLilePr6vlxyv6vr047ga0fCX5KHAzsxccX2mPADcl2T7O3pYb1+g7luQ7VXXGuPvQ8pTk\nv4Ffq6qfzKsfDzzkffTHjks3r3FJHjzSLsCPQtA4/Qx4M/DEvPrpbZ+OEYP+tW8V8PvAs/PqAf7z\n2Lcj/Z+rgL1JHuXl93icAZwF+P6OY8igf+27DXhTVT0wf0eSLx37dqRZVfXFJL8KnM8r/xj71UOf\nfaNjwzV6Seqcd91IUucMeknqnEEvSZ0z6CWpcwa9JHXufwGVHlMFmXXPLgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "joAmCCTuqm4R",
        "colab_type": "text"
      },
      "source": [
        "One way to deal with it: assign class weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nH_pN_x-lwvs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.utils import class_weight\n",
        "class_weights = np.array(class_weight.compute_class_weight('balanced', np.unique(y), y))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EO9oKPNlPqhi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "scaler = StandardScaler()\n",
        "X = scaler.fit_transform(df)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IotzoGAjNPlc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y.to_numpy(), test_size=0.3)\n",
        "\n",
        "input_size = X_train.shape[1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OyfpwmA-jzsO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data.dataset import Dataset\n",
        "\n",
        "class MyCustomDataset(Dataset):\n",
        "    def __init__(self, X, y):\n",
        "        assert len(X) == len(y)\n",
        "        self.X = X\n",
        "        self.y = y\n",
        "        \n",
        "    def __getitem__(self, index):\n",
        "        return (self.X[index], self.y[index])\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x6VH-Vi2THJi",
        "colab_type": "text"
      },
      "source": [
        "## Utils"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w3uPoHr8CTTm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "import pytorch_lightning as pl"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PNbITgo8FGo6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "thdmfmTBxVB0",
        "colab_type": "text"
      },
      "source": [
        "Lets create a class that would implement Deep Feature Selection"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YbS37RriDZQY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DFS(nn.Module):\n",
        "  def __init__(self, input_size):\n",
        "    super(DFS, self).__init__()\n",
        "    weights = torch.randn(input_size)\n",
        "    self.weight = nn.Parameter(weights)\n",
        "  \n",
        "  def forward(self, x):\n",
        "    return self.weight * x.type_as(self.weight)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UVCp0BjVXvmZ",
        "colab_type": "text"
      },
      "source": [
        "Lets create loss custom loss function. Also we add a class weights for case where we have imbalanced dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0iBQnjS-wZm5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MyCustomLoss(nn.Module):\n",
        "  def __init__(self, model, lambda1, lambda2, alpha1, alpha2, class_weights=None):\n",
        "    super().__init__()\n",
        "    self.model = model\n",
        "    self.lambda1 = lambda1\n",
        "    self.lambda2 = lambda2\n",
        "    self.alpha1 = alpha1\n",
        "    self.alpha2 = alpha2\n",
        "    self.class_weights = torch.tensor(class_weights, dtype=torch.float).to(device)\n",
        "    \n",
        "  def forward(self, output, target):\n",
        "    \"\"\"\n",
        "    This function make assumption that model[1] is DFS layer, and models[0] is Sequential\n",
        "    \"\"\"\n",
        "    target = target.view(-1,1)\n",
        "    standard_loss = nn.BCELoss(weight=self.class_weights[target.type(torch.LongTensor)])\n",
        "    loss = standard_loss(output, target)\n",
        "    \n",
        "    modules = self.model.modules()\n",
        "    \n",
        "    #ignore sequential\n",
        "    next(modules)\n",
        "    \n",
        "    #dfs loss\n",
        "    dfs = next(modules)\n",
        "    dfs_weights = dfs.weight\n",
        "    \n",
        "    loss += self.lambda1 * ((1 - self.lambda2) / 2 * (dfs_weights.norm(p=2) ** 2) +\n",
        "                            self.lambda2 * dfs_weights.abs().sum())\n",
        "    \n",
        "    #elastic-net-like for other layers\n",
        "    for module in modules:\n",
        "      if not hasattr(module, 'weight'):\n",
        "        continue\n",
        "      loss += self.alpha1 * ((1 - self.alpha2) / 2 * (module.weight.norm(p=2) ** 2) +\n",
        "                             self.alpha2 * module.weight.abs().sum())\n",
        "    self.loss = loss\n",
        "    return loss  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JKSE8iG7YgxI",
        "colab_type": "text"
      },
      "source": [
        "We need somehow evaluate our model. We can use metrics from sklearn, but this metrics for batch somewhat misleading. But if we evaluate on all validation data, we derive result that likely to be the truth"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uHG8DFwhjRs0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import precision_score, recall_score, accuracy_score, roc_auc_score, average_precision_score\n",
        "\n",
        "class Metric:\n",
        "  def __init__(self, name, function):\n",
        "    self.name = name\n",
        "    self.function = function\n",
        "    \n",
        "  def return_as_dict(self, value):\n",
        "    return {self.name: value}\n",
        "  \n",
        "  def calc(self, output: torch.Tensor, target: torch.Tensor):\n",
        "    return self.return_as_dict(torch.tensor([self.function(target.detach().cpu(), \n",
        "                                                           output.detach().cpu())]))\n",
        "  \n",
        "\n",
        "class ThresholdMetric(Metric):\n",
        "  def __init__(self, *args, threshold=0.5):\n",
        "    super().__init__(*args)\n",
        "    self.set_threshold(threshold)\n",
        "  \n",
        "  def set_threshold(self, threshold):\n",
        "    self.threshold = threshold\n",
        "    \n",
        "  def calc(self, output: torch.Tensor, target: torch.Tensor):\n",
        "    predicted_classes = output.gt(self.threshold).type(torch.LongTensor)\n",
        "    return super().calc(predicted_classes, target)\n",
        "\n",
        "  \n",
        "class Metrics:\n",
        "  def __init__(self, name, threshold=0.5):\n",
        "    \n",
        "    self.metrics = [\n",
        "        ThresholdMetric(name+'_precision', precision_score, threshold=threshold),\n",
        "        ThresholdMetric(name+'_recall', recall_score, threshold=threshold),\n",
        "        ThresholdMetric(name+'_accuracy', accuracy_score, threshold=threshold),\n",
        "        Metric(name+'_roc_auc', roc_auc_score),\n",
        "        Metric(name+'_AP', average_precision_score)\n",
        "    ]\n",
        "  \n",
        "  def calc(self, output, target):\n",
        "    result = list(map(lambda el: el.calc(output, target), self.metrics))\n",
        "    return {k: v for d in result for k, v in d.items()}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mMV7rCBVTKyo",
        "colab_type": "text"
      },
      "source": [
        "## Train nn"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kg_yi68Emof6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss_attrs = {\n",
        "    'alpha1': 0.01,\n",
        "    'alpha2': 0.01,\n",
        "    'lambda1': 0.01,\n",
        "    'lambda2': 0.01,\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R2kAgNJPW49S",
        "colab_type": "text"
      },
      "source": [
        "For convenient training our model I'll use [pytorch-lightning](https://github.com/williamFalcon/pytorch-lightning). It is a very lightweight wrapper on PyTorch and controls some things for you. And with this you can fastly build and train model. It also saves all metrics and loss in TensorBoard format."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4QyYDCBvl-M2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import DataLoader\n",
        "\n",
        "class CoolSystem(pl.LightningModule):\n",
        "  def __init__(self, loss_attrs, class_weights):\n",
        "    super().__init__()\n",
        "    \n",
        "    self.model = nn.Sequential(\n",
        "        DFS(input_size),\n",
        "        nn.Linear(input_size, 100),\n",
        "        nn.Linear(100, 1),\n",
        "        nn.Sigmoid()\n",
        "    )\n",
        "    \n",
        "    self.loss = MyCustomLoss(self.model, **loss_attrs, class_weights=class_weights)\n",
        "    self.train_metrics = Metrics('train')\n",
        "    self.val_metrics = Metrics('val')\n",
        "  \n",
        "  def forward(self, x):\n",
        "    return self.model(x)\n",
        "    \n",
        "  def training_step(self, batch, batch_nb):\n",
        "    x, y = batch\n",
        "    y_hat = self.forward(x)\n",
        "    return dict(loss=self.loss(y_hat, y), **self.train_metrics.calc(y_hat, y))\n",
        "    \n",
        "  def validation_step(self, batch, batch_nb):\n",
        "    x, y = batch\n",
        "    y_hat = self.forward(x)\n",
        "    return dict(val_loss=self.loss(y_hat, y), **self.val_metrics.calc(y_hat, y))\n",
        "  \n",
        "  def validation_end(self, outputs):\n",
        "    # OPTIONAL\n",
        "    print([dic['val_precision'] for dic in outputs])\n",
        "    v = {'avg_'+k: torch.stack([dic[k] for dic in outputs]).mean() for k in outputs[0]}\n",
        "    return v\n",
        "  \n",
        "  def configure_optimizers(self):\n",
        "    # REQUIRED\n",
        "    # can return multiple optimizers and learning_rate schedulers\n",
        "    return optim.Adam(self.model.parameters(), lr=1e-3)\n",
        "  \n",
        "  @pl.data_loader\n",
        "  def tng_dataloader(self):\n",
        "      # REQUIRED\n",
        "      return DataLoader(MyCustomDataset(X_train, y_train), batch_size=32)\n",
        "\n",
        "  @pl.data_loader\n",
        "  def val_dataloader(self):\n",
        "      # OPTIONAL\n",
        "      return DataLoader(MyCustomDataset(X_test, y_test), batch_size=32)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rX2knnKX43gv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "db4bc1d7-2b03-467f-b9a7-3975d13191ef"
      },
      "source": [
        "from pytorch_lightning import Trainer\n",
        "from test_tube import Experiment\n",
        "import os\n",
        "\n",
        "model = CoolSystem(loss_attrs=loss_attrs, class_weights=class_weights)\n",
        "\n",
        "# PyTorch summarywriter with a few bells and whistles    \n",
        "exp = Experiment(save_dir=os.getcwd())\n",
        "\n",
        "# train on cpu using only 10% of the data (for demo purposes)\n",
        "# pass in experiment for automatic tensorboard logging.    \n",
        "trainer = Trainer(experiment=exp, max_nb_epochs=20, gpus=[0])\n",
        "\n",
        "  \n",
        "trainer.fit(model)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 17%|█▋        | 6/36 [00:00<00:01, 18.42it/s, batch_nb=5, epoch=0, gpu=0, tng_loss=8.188, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "VISIBLE GPUS: '0'\n",
            "gpu available: True, used: True\n",
            "      Name          Type  Params\n",
            "0    model    Sequential  154125\n",
            "1  model.0           DFS    1524\n",
            "2  model.1        Linear  152500\n",
            "3  model.2        Linear     101\n",
            "4  model.3       Sigmoid       0\n",
            "5     loss  MyCustomLoss  154125\n",
            "[tensor([0.5556]), tensor([0.6667]), tensor([0.6000]), tensor([0.3333]), tensor([0.5714])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.18it/s, avg_valAP=0.948, avg_val_accuracy=0.745, avg_val_loss=7.65, avg_val_precision=0.925, avg_val_recall=0.702, avg_val_roc_auc=0.882, batch_nb=24, epoch=0, gpu=0, tng_loss=7.936, v_nb=21]\n",
            " 17%|█▋        | 6/36 [00:00<00:01, 17.76it/s, avg_valAP=0.948, avg_val_accuracy=0.745, avg_val_loss=7.65, avg_val_precision=0.925, avg_val_recall=0.702, avg_val_roc_auc=0.882, batch_nb=5, epoch=1, gpu=0, tng_loss=7.873, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9444]), tensor([0.9000]), tensor([1.]), tensor([0.8125]), tensor([0.8000]), tensor([0.9375]), tensor([1.]), tensor([1.]), tensor([0.7826]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 55.01it/s, avg_valAP=0.971, avg_val_accuracy=0.824, avg_val_loss=7.23, avg_val_precision=0.958, avg_val_recall=0.789, avg_val_roc_auc=0.929, batch_nb=24, epoch=1, gpu=0, tng_loss=7.670, v_nb=21]\n",
            " 14%|█▍        | 5/36 [00:00<00:02, 15.45it/s, avg_valAP=0.971, avg_val_accuracy=0.824, avg_val_loss=7.23, avg_val_precision=0.958, avg_val_recall=0.789, avg_val_roc_auc=0.929, batch_nb=4, epoch=2, gpu=0, tng_loss=7.624, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([0.8889]), tensor([1.]), tensor([0.8824]), tensor([0.9474]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 53.59it/s, avg_valAP=0.977, avg_val_accuracy=0.835, avg_val_loss=6.91, avg_val_precision=0.968, avg_val_recall=0.797, avg_val_roc_auc=0.944, batch_nb=24, epoch=2, gpu=0, tng_loss=7.460, v_nb=21]\n",
            " 14%|█▍        | 5/36 [00:00<00:01, 15.56it/s, avg_valAP=0.977, avg_val_accuracy=0.835, avg_val_loss=6.91, avg_val_precision=0.968, avg_val_recall=0.797, avg_val_roc_auc=0.944, batch_nb=4, epoch=3, gpu=0, tng_loss=7.422, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([1.]), tensor([0.9565]), tensor([0.8947]), tensor([1.]), tensor([0.8824]), tensor([1.]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 53.81it/s, avg_valAP=0.977, avg_val_accuracy=0.844, avg_val_loss=6.63, avg_val_precision=0.973, avg_val_recall=0.806, avg_val_roc_auc=0.945, batch_nb=24, epoch=3, gpu=0, tng_loss=7.276, v_nb=21]\n",
            " 17%|█▋        | 6/36 [00:00<00:01, 16.83it/s, avg_valAP=0.977, avg_val_accuracy=0.844, avg_val_loss=6.63, avg_val_precision=0.973, avg_val_recall=0.806, avg_val_roc_auc=0.945, batch_nb=5, epoch=4, gpu=0, tng_loss=7.180, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([1.]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([1.]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 57.71it/s, avg_valAP=0.976, avg_val_accuracy=0.844, avg_val_loss=6.36, avg_val_precision=0.968, avg_val_recall=0.81, avg_val_roc_auc=0.945, batch_nb=24, epoch=4, gpu=0, tng_loss=6.903, v_nb=21]\n",
            " 17%|█▋        | 6/36 [00:00<00:01, 17.31it/s, avg_valAP=0.976, avg_val_accuracy=0.844, avg_val_loss=6.36, avg_val_precision=0.968, avg_val_recall=0.81, avg_val_roc_auc=0.945, batch_nb=5, epoch=5, gpu=0, tng_loss=6.825, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9545]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([1.]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.52it/s, avg_valAP=0.977, avg_val_accuracy=0.852, avg_val_loss=6.11, avg_val_precision=0.968, avg_val_recall=0.822, avg_val_roc_auc=0.946, batch_nb=24, epoch=5, gpu=0, tng_loss=6.598, v_nb=21]\n",
            " 17%|█▋        | 6/36 [00:00<00:01, 17.30it/s, avg_valAP=0.977, avg_val_accuracy=0.852, avg_val_loss=6.11, avg_val_precision=0.968, avg_val_recall=0.822, avg_val_roc_auc=0.946, batch_nb=5, epoch=6, gpu=0, tng_loss=6.530, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9545]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([1.]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 56.50it/s, avg_valAP=0.977, avg_val_accuracy=0.852, avg_val_loss=5.88, avg_val_precision=0.968, avg_val_recall=0.822, avg_val_roc_auc=0.946, batch_nb=24, epoch=6, gpu=0, tng_loss=6.322, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 18.03it/s, avg_valAP=0.977, avg_val_accuracy=0.852, avg_val_loss=5.88, avg_val_precision=0.968, avg_val_recall=0.822, avg_val_roc_auc=0.946, batch_nb=6, epoch=7, gpu=0, tng_loss=6.248, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9545]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([1.]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 57.81it/s, avg_valAP=0.977, avg_val_accuracy=0.849, avg_val_loss=5.65, avg_val_precision=0.963, avg_val_recall=0.822, avg_val_roc_auc=0.946, batch_nb=24, epoch=7, gpu=0, tng_loss=6.066, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 18.25it/s, avg_valAP=0.977, avg_val_accuracy=0.849, avg_val_loss=5.65, avg_val_precision=0.963, avg_val_recall=0.822, avg_val_roc_auc=0.946, batch_nb=6, epoch=8, gpu=0, tng_loss=5.997, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9545]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([0.9444]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.18it/s, avg_valAP=0.977, avg_val_accuracy=0.855, avg_val_loss=5.44, avg_val_precision=0.963, avg_val_recall=0.83, avg_val_roc_auc=0.946, batch_nb=24, epoch=8, gpu=0, tng_loss=5.825, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 17.97it/s, avg_valAP=0.977, avg_val_accuracy=0.855, avg_val_loss=5.44, avg_val_precision=0.963, avg_val_recall=0.83, avg_val_roc_auc=0.946, batch_nb=6, epoch=9, gpu=0, tng_loss=5.760, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9545]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([0.9444]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.77it/s, avg_valAP=0.976, avg_val_accuracy=0.86, avg_val_loss=5.24, avg_val_precision=0.955, avg_val_recall=0.844, avg_val_roc_auc=0.944, batch_nb=24, epoch=9, gpu=0, tng_loss=5.597, v_nb=21]\n",
            " 17%|█▋        | 6/36 [00:00<00:01, 17.85it/s, avg_valAP=0.976, avg_val_accuracy=0.86, avg_val_loss=5.24, avg_val_precision=0.955, avg_val_recall=0.844, avg_val_roc_auc=0.944, batch_nb=5, epoch=10, gpu=0, tng_loss=5.544, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9130]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([0.8947]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.41it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=5.05, avg_val_precision=0.955, avg_val_recall=0.844, avg_val_roc_auc=0.941, batch_nb=24, epoch=10, gpu=0, tng_loss=5.381, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 18.01it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=5.05, avg_val_precision=0.955, avg_val_recall=0.844, avg_val_roc_auc=0.941, batch_nb=6, epoch=11, gpu=0, tng_loss=5.323, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9130]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([0.8947]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.37it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=4.87, avg_val_precision=0.955, avg_val_recall=0.844, avg_val_roc_auc=0.942, batch_nb=24, epoch=11, gpu=0, tng_loss=5.176, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 18.12it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=4.87, avg_val_precision=0.955, avg_val_recall=0.844, avg_val_roc_auc=0.942, batch_nb=6, epoch=12, gpu=0, tng_loss=5.120, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9130]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([0.8947]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.58it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=4.7, avg_val_precision=0.955, avg_val_recall=0.844, avg_val_roc_auc=0.94, batch_nb=24, epoch=12, gpu=0, tng_loss=4.981, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 17.83it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=4.7, avg_val_precision=0.955, avg_val_recall=0.844, avg_val_roc_auc=0.94, batch_nb=6, epoch=13, gpu=0, tng_loss=4.928, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9130]), tensor([0.9565]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([0.8947]), tensor([1.]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.09it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=4.54, avg_val_precision=0.951, avg_val_recall=0.847, avg_val_roc_auc=0.94, batch_nb=24, epoch=13, gpu=0, tng_loss=4.794, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 17.96it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=4.54, avg_val_precision=0.951, avg_val_recall=0.847, avg_val_roc_auc=0.94, batch_nb=6, epoch=14, gpu=0, tng_loss=4.744, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9130]), tensor([0.9583]), tensor([0.9444]), tensor([1.]), tensor([0.8889]), tensor([0.8947]), tensor([0.9500]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.56it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=4.38, avg_val_precision=0.947, avg_val_recall=0.851, avg_val_roc_auc=0.94, batch_nb=24, epoch=14, gpu=0, tng_loss=4.617, v_nb=21]\n",
            " 17%|█▋        | 6/36 [00:00<00:01, 16.65it/s, avg_valAP=0.974, avg_val_accuracy=0.86, avg_val_loss=4.38, avg_val_precision=0.947, avg_val_recall=0.851, avg_val_roc_auc=0.94, batch_nb=5, epoch=15, gpu=0, tng_loss=4.576, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9167]), tensor([0.9583]), tensor([0.8947]), tensor([1.]), tensor([0.8889]), tensor([0.8947]), tensor([0.9500]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 19%|█▉        | 7/36 [00:00<00:01, 18.83it/s, avg_valAP=0.974, avg_val_accuracy=0.868, avg_val_loss=4.23, avg_val_precision=0.952, avg_val_recall=0.86, avg_val_roc_auc=0.941, batch_nb=6, epoch=16, gpu=0, tng_loss=4.402, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9167]), tensor([0.9583]), tensor([0.8947]), tensor([1.]), tensor([0.8947]), tensor([0.9444]), tensor([0.9500]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.38it/s, avg_valAP=0.974, avg_val_accuracy=0.866, avg_val_loss=4.08, avg_val_precision=0.948, avg_val_recall=0.86, avg_val_roc_auc=0.94, batch_nb=24, epoch=16, gpu=0, tng_loss=4.286, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 17.56it/s, avg_valAP=0.974, avg_val_accuracy=0.866, avg_val_loss=4.08, avg_val_precision=0.948, avg_val_recall=0.86, avg_val_roc_auc=0.94, batch_nb=6, epoch=17, gpu=0, tng_loss=4.243, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9167]), tensor([0.9583]), tensor([0.8947]), tensor([1.]), tensor([0.8947]), tensor([0.9444]), tensor([0.9048]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 58.56it/s, avg_valAP=0.973, avg_val_accuracy=0.868, avg_val_loss=3.94, avg_val_precision=0.948, avg_val_recall=0.864, avg_val_roc_auc=0.939, batch_nb=24, epoch=17, gpu=0, tng_loss=4.132, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 18.27it/s, avg_valAP=0.973, avg_val_accuracy=0.868, avg_val_loss=3.94, avg_val_precision=0.948, avg_val_recall=0.864, avg_val_roc_auc=0.939, batch_nb=6, epoch=18, gpu=0, tng_loss=4.092, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9200]), tensor([0.9583]), tensor([0.8947]), tensor([1.]), tensor([0.8947]), tensor([0.9444]), tensor([0.9048]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 57.94it/s, avg_valAP=0.973, avg_val_accuracy=0.868, avg_val_loss=3.81, avg_val_precision=0.948, avg_val_recall=0.864, avg_val_roc_auc=0.938, batch_nb=24, epoch=18, gpu=0, tng_loss=3.987, v_nb=21]\n",
            " 19%|█▉        | 7/36 [00:00<00:01, 18.01it/s, avg_valAP=0.973, avg_val_accuracy=0.868, avg_val_loss=3.81, avg_val_precision=0.948, avg_val_recall=0.864, avg_val_roc_auc=0.938, batch_nb=6, epoch=19, gpu=0, tng_loss=3.948, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9200]), tensor([0.9583]), tensor([0.8947]), tensor([1.]), tensor([0.8947]), tensor([0.9444]), tensor([0.9048]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 36/36 [00:00<00:00, 26.83it/s, avg_valAP=0.973, avg_val_accuracy=0.868, avg_val_loss=3.68, avg_val_precision=0.948, avg_val_recall=0.864, avg_val_roc_auc=0.939, batch_nb=24, epoch=19, gpu=0, tng_loss=3.849, v_nb=21]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[tensor([1.]), tensor([0.9200]), tensor([0.9583]), tensor([0.8947]), tensor([1.]), tensor([0.8947]), tensor([0.9444]), tensor([0.9048]), tensor([1.]), tensor([0.9091]), tensor([1.])]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2RFTMGHx_qGL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!rm -rf /gdrive/My\\ Drive/TensorBoard\n",
        "!cp -r default/ /gdrive/My\\ Drive/TensorBoard"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "56nsImEZOEsz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        },
        "outputId": "d2ba6fec-1d4f-4be7-cd5d-263a38368559"
      },
      "source": [
        "from pprint import pprint\n",
        "predicted = model(torch.tensor(X_test))\n",
        "pprint(Metrics('NN').calc(predicted, torch.tensor(y_test)))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'NNAP': tensor([0.9507]),\n",
            " 'NN_accuracy': tensor([0.8554]),\n",
            " 'NN_precision': tensor([0.9469]),\n",
            " 'NN_recall': tensor([0.8412]),\n",
            " 'NN_roc_auc': tensor([0.9076])}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X8vMUxMQzG2_",
        "colab_type": "text"
      },
      "source": [
        "## RandomForests approach comparision"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ANMGQ4kjy_dY",
        "colab_type": "code",
        "outputId": "0c81f67f-4cfb-4c35-aae1-36ff04f3eed5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        }
      },
      "source": [
        "%%time\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "rfc = RandomForestClassifier(class_weight='balanced')\n",
        "rfc.fit(X_train, y_train)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 131 ms, sys: 10.2 ms, total: 141 ms\n",
            "Wall time: 146 ms\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/ensemble/forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
            "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-G3i6zr1zXzo",
        "colab_type": "code",
        "outputId": "b2130bbf-3f1a-40bf-e9e0-19cdeab8ba68",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        }
      },
      "source": [
        "predicted = rfc.predict(X_test)\n",
        "pprint(Metrics('RF').calc(torch.tensor(predicted), torch.tensor(y_test)))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'RFAP': tensor([0.8607]),\n",
            " 'RF_accuracy': tensor([0.8524]),\n",
            " 'RF_precision': tensor([0.8740]),\n",
            " 'RF_recall': tensor([0.9227]),\n",
            " 'RF_roc_auc': tensor([0.8048])}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iWtDCva1VKQa",
        "colab_type": "text"
      },
      "source": [
        "Thus we can conclude, that in general Neural Network with DFS better than RandomForests, but worse only in recall"
      ]
    }
  ]
}