62-modelo-genero-nome-pyspark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "5dff34ff-1961-4769-ba8a-9d63be4a0e2d",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">openjdk version &#34;1.8.0_275&#34;\r\n",
       "OpenJDK Runtime Environment (Zulu 8.50.0.51-CA-linux64) (build 1.8.0_275-b01)\r\n",
       "OpenJDK 64-Bit Server VM (Zulu 8.50.0.51-CA-linux64) (build 25.275-b01, mixed mode)\r\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">openjdk version &#34;1.8.0_275&#34;\r\nOpenJDK Runtime Environment (Zulu 8.50.0.51-CA-linux64) (build 1.8.0_275-b01)\r\nOpenJDK 64-Bit Server VM (Zulu 8.50.0.51-CA-linux64) (build 25.275-b01, mixed mode)\r\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "!java -version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "129af4d4-0a5b-40bc-a4a7-92f623441ecc",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[87]: &#39;3.1.0&#39;</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[87]: &#39;3.1.0&#39;</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sc.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "6351029a-f885-4ea0-805d-2b7eba18c603",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#!pip install mlflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "fb339128-b151-4cc3-a414-ccda5baa0ad3",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#import math\n",
    "#import pandas as pd\n",
    "#import matplotlib.pylab as plt\n",
    "#%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "098e35dc-3fff-4e5a-a03f-bacec49db898",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# carregar tabela de dados\n",
    "df = table(\"cadastro_nomes_sexo\")\n",
    "#df = table(\"nome_sexo_pf50\")\n",
    "#df = table(\"nome_sexo_pf05\")\n",
    "#df = spark.sql(\"select * from cadastro_nomes_sexo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "771c6b06-daed-4c82-9124-3942dcb3ab13",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">root\n",
       "-- PNOME: string (nullable = true)\n",
       "-- SEXO: string (nullable = true)\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">root\n |-- PNOME: string (nullable = true)\n |-- SEXO: string (nullable = true)\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "20063a66-19ca-400c-ba25-056480f82522",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[92]: 48970</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[92]: 48970</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "740d1007-0d7a-4516-9312-883a1a240716",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+---------+----+\n",
       "    PNOME|SEXO|\n",
       "+---------+----+\n",
       "    MARIA|   F|\n",
       "     JOSE|   M|\n",
       "      ANA|   F|\n",
       "     JOAO|   M|\n",
       "   CARLOS|   M|\n",
       "    PAULO|   M|\n",
       "  ANTONIO|   M|\n",
       "     LUIZ|   M|\n",
       "FRANCISCO|   M|\n",
       "  MARCELO|   M|\n",
       "+---------+----+\n",
       "only showing top 10 rows\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">+---------+----+\n|    PNOME|SEXO|\n+---------+----+\n|    MARIA|   F|\n|     JOSE|   M|\n|      ANA|   F|\n|     JOAO|   M|\n|   CARLOS|   M|\n|    PAULO|   M|\n|  ANTONIO|   M|\n|     LUIZ|   M|\n|FRANCISCO|   M|\n|  MARCELO|   M|\n+---------+----+\nonly showing top 10 rows\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "33c42334-8994-4180-9cc4-502326727ff3",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "col_nome = df.columns[0] # PNOME\n",
    "col_sexo = df.columns[1] # SEXO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "27a4da81-02ef-4213-a70a-c9e55476cf75",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[95]: PythonRDD[3561] at RDD at PythonRDD.scala:58</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[95]: PythonRDD[3561] at RDD at PythonRDD.scala:58</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rdd = df.rdd.filter(lambda x: x[col_sexo] in ['M', 'F'] and x[col_nome] != None)\n",
    "#rdd = df.rdd\n",
    "#rdd = df.rdd.map(list)\n",
    "#rdd = df.rdd.map(tuple)\n",
    "rdd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "a441b331-92f3-47a8-bbf4-dbb62bf97c66",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[96]: pyspark.rdd.PipelinedRDD</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[96]: pyspark.rdd.PipelinedRDD</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "type(rdd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "51e0d99e-1acd-4dbc-baea-5b46e735b5c6",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[97]: [Row(PNOME=&#39;MARIA&#39;, SEXO=&#39;F&#39;),\n",
       " Row(PNOME=&#39;JOSE&#39;, SEXO=&#39;M&#39;),\n",
       " Row(PNOME=&#39;ANA&#39;, SEXO=&#39;F&#39;),\n",
       " Row(PNOME=&#39;JOAO&#39;, SEXO=&#39;M&#39;),\n",
       " Row(PNOME=&#39;CARLOS&#39;, SEXO=&#39;M&#39;)]</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[97]: [Row(PNOME=&#39;MARIA&#39;, SEXO=&#39;F&#39;),\n Row(PNOME=&#39;JOSE&#39;, SEXO=&#39;M&#39;),\n Row(PNOME=&#39;ANA&#39;, SEXO=&#39;F&#39;),\n Row(PNOME=&#39;JOAO&#39;, SEXO=&#39;M&#39;),\n Row(PNOME=&#39;CARLOS&#39;, SEXO=&#39;M&#39;)]</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rdd.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "c4b837d2-11cd-4d58-a8a4-34e90490e238",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[98]: PythonRDD[3563] at RDD at PythonRDD.scala:58</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[98]: PythonRDD[3563] at RDD at PythonRDD.scala:58</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.sql import Row\n",
    "\n",
    "rdd2 = rdd.map(lambda x: Row(\n",
    "  nome=x[col_nome],\n",
    "  sexo={'F': 0, 'M': 1, 'X': 9}[x[col_sexo]]\n",
    "))\n",
    "rdd2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "258cd10b-7b6e-4bb3-9e20-a7e5c2d35420",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[99]: [Row(nome=&#39;MARIA&#39;, sexo=0),\n",
       " Row(nome=&#39;JOSE&#39;, sexo=1),\n",
       " Row(nome=&#39;ANA&#39;, sexo=0),\n",
       " Row(nome=&#39;JOAO&#39;, sexo=1),\n",
       " Row(nome=&#39;CARLOS&#39;, sexo=1)]</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[99]: [Row(nome=&#39;MARIA&#39;, sexo=0),\n Row(nome=&#39;JOSE&#39;, sexo=1),\n Row(nome=&#39;ANA&#39;, sexo=0),\n Row(nome=&#39;JOAO&#39;, sexo=1),\n Row(nome=&#39;CARLOS&#39;, sexo=1)]</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rdd2.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "bd5ccac6-f7ec-4c7d-8f0e-018d11b1e0ee",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "MAIOR_QTDE_LETRAS = 16 # fixado para o maior arquivo\n",
    "\n",
    "def incluir_letras(row):\n",
    "  row_dict = row.asDict()\n",
    "  rev = row_dict['nome'][::-1]\n",
    "  tam = len(rev)\n",
    "  letras = [0] * MAIOR_QTDE_LETRAS\n",
    "  \n",
    "  for i in range(tam):\n",
    "    letras[i] = ord(rev[i]) - 64\n",
    "  row_dict['letras'] = letras\n",
    "  \n",
    "  new_row = Row(**row_dict)\n",
    "  return new_row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "46daac5b-32b7-465b-a1e3-94a8b6e8ecf3",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[101]: PythonRDD[3565] at RDD at PythonRDD.scala:58</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[101]: PythonRDD[3565] at RDD at PythonRDD.scala:58</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rdd3 = rdd2.map(incluir_letras)\n",
    "rdd3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "36a5574a-cbf0-4aa6-8a13-f4d13a549bb7",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[102]: 41150</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[102]: 41150</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rdd3.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "ceb1545d-cf96-4418-a786-110968e34cbd",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[103]: [Row(nome=&#39;MARIA&#39;, sexo=0, letras=[1, 9, 18, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " Row(nome=&#39;JOSE&#39;, sexo=1, letras=[5, 19, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " Row(nome=&#39;ANA&#39;, sexo=0, letras=[1, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " Row(nome=&#39;JOAO&#39;, sexo=1, letras=[15, 1, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " Row(nome=&#39;CARLOS&#39;, sexo=1, letras=[19, 15, 12, 18, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])]</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[103]: [Row(nome=&#39;MARIA&#39;, sexo=0, letras=[1, 9, 18, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n Row(nome=&#39;JOSE&#39;, sexo=1, letras=[5, 19, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n Row(nome=&#39;ANA&#39;, sexo=0, letras=[1, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n Row(nome=&#39;JOAO&#39;, sexo=1, letras=[15, 1, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n Row(nome=&#39;CARLOS&#39;, sexo=1, letras=[19, 15, 12, 18, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])]</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rdd3.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "a065e561-4e43-4d4c-8617-9a3c672cb5f8",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[104]: DataFrame[nome: string, sexo: bigint, letras: array&lt;bigint&gt;]</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[104]: DataFrame[nome: string, sexo: bigint, letras: array&lt;bigint&gt;]</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df3 = rdd3.toDF()\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "39c7f4b0-2573-450f-85a3-b3f72be41544",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">root\n",
       "-- nome: string (nullable = true)\n",
       "-- sexo: long (nullable = true)\n",
       "-- letras: array (nullable = true)\n",
       "    |-- element: long (containsNull = true)\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">root\n |-- nome: string (nullable = true)\n |-- sexo: long (nullable = true)\n |-- letras: array (nullable = true)\n |    |-- element: long (containsNull = true)\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df3.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "03bd013a-937f-4dc7-9b33-abe992fe7aa5",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+-------+------+------------------+\n",
       "summary|  nome|              sexo|\n",
       "+-------+------+------------------+\n",
       "  count| 41150|             41150|\n",
       "   mean|   NaN|0.4680194410692588|\n",
       " stddev|  null|0.4989822586799242|\n",
       "    min| AARAO|                 0|\n",
       "    max|ZYSMAN|                 1|\n",
       "+-------+------+------------------+\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">+-------+------+------------------+\n|summary|  nome|              sexo|\n+-------+------+------------------+\n|  count| 41150|             41150|\n|   mean|   NaN|0.4680194410692588|\n| stddev|  null|0.4989822586799242|\n|    min| AARAO|                 0|\n|    max|ZYSMAN|                 1|\n+-------+------+------------------+\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df3.select(\"nome\", \"sexo\").describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "6429ddf6-7e59-4647-a8f7-de2572052e79",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml.linalg import Vectors\n",
    "\n",
    "def converter_para_ponto_rotulado(row):\n",
    "  vec = Vectors.dense(row[\"letras\"])\n",
    "  #vec = Vectors.sparse(MAIOR_QTDE_LETRAS, row[\"letras\"])\n",
    "  obj = (row[\"nome\"], row[\"sexo\"], vec)\n",
    "  return obj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "4582b9c4-6496-49f5-93c6-f4a38018ada7",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[125]: PythonRDD[3583] at RDD at PythonRDD.scala:58</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[125]: PythonRDD[3583] at RDD at PythonRDD.scala:58</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rdd4 = rdd3.map(converter_para_ponto_rotulado)\n",
    "rdd4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "5ffb4a73-0ca4-4ee4-9acd-e774ae6088b8",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[126]: [(&#39;MARIA&#39;,\n",
       "  0,\n",
       "  DenseVector([1.0, 9.0, 18.0, 1.0, 13.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])),\n",
       " (&#39;JOSE&#39;,\n",
       "  1,\n",
       "  DenseVector([5.0, 19.0, 15.0, 10.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])),\n",
       " (&#39;ANA&#39;,\n",
       "  0,\n",
       "  DenseVector([1.0, 14.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])),\n",
       " (&#39;JOAO&#39;,\n",
       "  1,\n",
       "  DenseVector([15.0, 1.0, 15.0, 10.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])),\n",
       " (&#39;CARLOS&#39;,\n",
       "  1,\n",
       "  DenseVector([19.0, 15.0, 12.0, 18.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]))]</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[126]: [(&#39;MARIA&#39;,\n  0,\n  DenseVector([1.0, 9.0, 18.0, 1.0, 13.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])),\n (&#39;JOSE&#39;,\n  1,\n  DenseVector([5.0, 19.0, 15.0, 10.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])),\n (&#39;ANA&#39;,\n  0,\n  DenseVector([1.0, 14.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])),\n (&#39;JOAO&#39;,\n  1,\n  DenseVector([15.0, 1.0, 15.0, 10.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])),\n (&#39;CARLOS&#39;,\n  1,\n  DenseVector([19.0, 15.0, 12.0, 18.0, 1.0, 3.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]))]</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rdd4.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "c623c57f-44fc-49c7-b66b-60f704aa5653",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[127]: DataFrame[nome: string, real: bigint, features: vector]</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[127]: DataFrame[nome: string, real: bigint, features: vector]</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df4 = rdd4.toDF([\"nome\", \"real\", \"features\"])\n",
    "df4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "def3279e-89ef-43b9-99fa-d75074c8e1cc",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+------+----+---------------------------------------------------------------------+\n",
       "nome  |real|features                                                             |\n",
       "+------+----+---------------------------------------------------------------------+\n",
       "MARIA |0   |[1.0,9.0,18.0,1.0,13.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]  |\n",
       "JOSE  |1   |[5.0,19.0,15.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] |\n",
       "ANA   |0   |[1.0,14.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]   |\n",
       "JOAO  |1   |[15.0,1.0,15.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] |\n",
       "CARLOS|1   |[19.0,15.0,12.0,18.0,1.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]|\n",
       "+------+----+---------------------------------------------------------------------+\n",
       "only showing top 5 rows\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">+------+----+---------------------------------------------------------------------+\n|nome  |real|features                                                             |\n+------+----+---------------------------------------------------------------------+\n|MARIA |0   |[1.0,9.0,18.0,1.0,13.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]  |\n|JOSE  |1   |[5.0,19.0,15.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] |\n|ANA   |0   |[1.0,14.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]   |\n|JOAO  |1   |[15.0,1.0,15.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] |\n|CARLOS|1   |[19.0,15.0,12.0,18.0,1.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]|\n+------+----+---------------------------------------------------------------------+\nonly showing top 5 rows\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df4.show(5, truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "0db7f2fc-734d-40ee-be05-cc87c9def90c",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml.feature import PCA\n",
    "\n",
    "pca = PCA(\n",
    "  k = 10,\n",
    "  inputCol = \"features\",\n",
    "  outputCol = \"pcaFeatures\"\n",
    ")\n",
    "\n",
    "#pcaModel = pca.fit(df4)\n",
    "#pcaResult = pcaModel.transform(df4).select(\"nome\", \"real\", \"pcaFeatures\")\n",
    "#pcaResult.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "c6f4f72c-d890-4cf8-8194-174f39d44261",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+------+----+---------------------------------------------------------------------+\n",
       "nome  |real|features                                                             |\n",
       "+------+----+---------------------------------------------------------------------+\n",
       "MARIA |0   |[1.0,9.0,18.0,1.0,13.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]  |\n",
       "JOSE  |1   |[5.0,19.0,15.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] |\n",
       "ANA   |0   |[1.0,14.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]   |\n",
       "JOAO  |1   |[15.0,1.0,15.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] |\n",
       "CARLOS|1   |[19.0,15.0,12.0,18.0,1.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]|\n",
       "+------+----+---------------------------------------------------------------------+\n",
       "only showing top 5 rows\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">+------+----+---------------------------------------------------------------------+\n|nome  |real|features                                                             |\n+------+----+---------------------------------------------------------------------+\n|MARIA |0   |[1.0,9.0,18.0,1.0,13.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]  |\n|JOSE  |1   |[5.0,19.0,15.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] |\n|ANA   |0   |[1.0,14.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]   |\n|JOAO  |1   |[15.0,1.0,15.0,10.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] |\n|CARLOS|1   |[19.0,15.0,12.0,18.0,1.0,3.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]|\n+------+----+---------------------------------------------------------------------+\nonly showing top 5 rows\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# usar redução de dimensionalidade PCA\n",
    "#df5 = pcaResult.withColumnRenamed(\"pcaFeatures\", \"features\")\n",
    "\n",
    "# não usar PCA\n",
    "df5 = df4\n",
    "\n",
    "df5.show(5, truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "ff96f724-66ab-4e1f-bc44-69440947ce86",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">dados de treino: 28895\n",
       "dados de teste:  12255\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">dados de treino: 28895\ndados de teste:  12255\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dividir dados entre treino e teste\n",
    "(dados_treino, dados_teste) = df5.randomSplit([0.7, 0.3], seed=42)\n",
    "\n",
    "dados_treino.cache()\n",
    "dados_teste.cache()\n",
    "df5.cache()\n",
    "\n",
    "print(\"dados de treino:\", dados_treino.count())\n",
    "print(\"dados de teste: \", dados_teste.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "5df7c79a-ee95-4339-8e73-865f3d71606a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[132]: &#39;from pyspark.ml.stat import Correlation\\n\\nr1 = Correlation.corr(dados_treino, &#34;features&#34;).head()\\nprint(&#34;Pearson correlation matrix:\\n&#34; + str(r1[0]))\\n\\nr2 = Correlation.corr(dados_treino, &#34;features&#34;, &#34;spearman&#34;).head()\\nprint(&#34;Spearman correlation matrix:\\n&#34; + str(r2[0]))&#39;</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[132]: &#39;from pyspark.ml.stat import Correlation\\n\\nr1 = Correlation.corr(dados_treino, &#34;features&#34;).head()\\nprint(&#34;Pearson correlation matrix:\\n&#34; + str(r1[0]))\\n\\nr2 = Correlation.corr(dados_treino, &#34;features&#34;, &#34;spearman&#34;).head()\\nprint(&#34;Spearman correlation matrix:\\n&#34; + str(r2[0]))&#39;</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# https://spark.apache.org/docs/latest/ml-statistics.html\n",
    "\n",
    "'''from pyspark.ml.stat import Correlation\n",
    "\n",
    "r1 = Correlation.corr(dados_treino, \"features\").head()\n",
    "print(\"Pearson correlation matrix:\\n\" + str(r1[0]))\n",
    "\n",
    "r2 = Correlation.corr(dados_treino, \"features\", \"spearman\").head()\n",
    "print(\"Spearman correlation matrix:\\n\" + str(r2[0]))'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "c88e5292-3423-4ce5-8804-2104e4f3d3ae",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[133]: &#39;from pyspark.ml.stat import ChiSquareTest\\n\\nr = ChiSquareTest.test(dados_treino, &#34;features&#34;, &#34;real&#34;).head()\\nprint(&#34;pValues: &#34; + str(r.pValues))\\nprint(&#34;degreesOfFreedom: &#34; + str(r.degreesOfFreedom))\\nprint(&#34;statistics: &#34; + str(r.statistics))&#39;</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[133]: &#39;from pyspark.ml.stat import ChiSquareTest\\n\\nr = ChiSquareTest.test(dados_treino, &#34;features&#34;, &#34;real&#34;).head()\\nprint(&#34;pValues: &#34; + str(r.pValues))\\nprint(&#34;degreesOfFreedom: &#34; + str(r.degreesOfFreedom))\\nprint(&#34;statistics: &#34; + str(r.statistics))&#39;</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''from pyspark.ml.stat import ChiSquareTest\n",
    "\n",
    "r = ChiSquareTest.test(dados_treino, \"features\", \"real\").head()\n",
    "print(\"pValues: \" + str(r.pValues))\n",
    "print(\"degreesOfFreedom: \" + str(r.degreesOfFreedom))\n",
    "print(\"statistics: \" + str(r.statistics))'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "c055fcbc-f3b1-4429-94d9-39f614000560",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[134]: &#39;from pyspark.ml.stat import Summarizer\\n\\n# create summarizer for multiple metrics &#34;mean&#34; and &#34;count&#34;\\nsummarizer = Summarizer.metrics(&#34;mean&#34;, &#34;count&#34;)\\n\\n# compute statistics for multiple metrics with weight\\ndados_treino.select(summarizer.summary(dados_treino.features, dados_treino.real)).show(truncate=False)\\n\\n# compute statistics for multiple metrics without weight\\ndados_treino.select(summarizer.summary(dados_treino.features)).show(truncate=False)\\n\\n# compute statistics for single metric &#34;mean&#34; with weight\\ndados_treino.select(Summarizer.mean(dados_treino.features, dados_treino.real)).show(truncate=False)\\n\\n# compute statistics for single metric &#34;mean&#34; without weight\\ndados_treino.select(Summarizer.mean(dados_treino.features)).show(truncate=False)&#39;</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[134]: &#39;from pyspark.ml.stat import Summarizer\\n\\n# create summarizer for multiple metrics &#34;mean&#34; and &#34;count&#34;\\nsummarizer = Summarizer.metrics(&#34;mean&#34;, &#34;count&#34;)\\n\\n# compute statistics for multiple metrics with weight\\ndados_treino.select(summarizer.summary(dados_treino.features, dados_treino.real)).show(truncate=False)\\n\\n# compute statistics for multiple metrics without weight\\ndados_treino.select(summarizer.summary(dados_treino.features)).show(truncate=False)\\n\\n# compute statistics for single metric &#34;mean&#34; with weight\\ndados_treino.select(Summarizer.mean(dados_treino.features, dados_treino.real)).show(truncate=False)\\n\\n# compute statistics for single metric &#34;mean&#34; without weight\\ndados_treino.select(Summarizer.mean(dados_treino.features)).show(truncate=False)&#39;</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''from pyspark.ml.stat import Summarizer\n",
    "\n",
    "# create summarizer for multiple metrics \"mean\" and \"count\"\n",
    "summarizer = Summarizer.metrics(\"mean\", \"count\")\n",
    "\n",
    "# compute statistics for multiple metrics with weight\n",
    "dados_treino.select(summarizer.summary(dados_treino.features, dados_treino.real)).show(truncate=False)\n",
    "\n",
    "# compute statistics for multiple metrics without weight\n",
    "dados_treino.select(summarizer.summary(dados_treino.features)).show(truncate=False)\n",
    "\n",
    "# compute statistics for single metric \"mean\" with weight\n",
    "dados_treino.select(Summarizer.mean(dados_treino.features, dados_treino.real)).show(truncate=False)\n",
    "\n",
    "# compute statistics for single metric \"mean\" without weight\n",
    "dados_treino.select(Summarizer.mean(dados_treino.features)).show(truncate=False)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "05cf531b-ee35-4a38-940f-f0df8590ae3c",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "models = {}\n",
    "\n",
    "def evaluate_model(name, classifier, evaluator, train_data, test_data):\n",
    "\n",
    "  start = datetime.now()\n",
    "  model = classifier.fit(train_data)\n",
    "  end = datetime.now()\n",
    "  elapsed = int((end - start).total_seconds() * 1000)\n",
    "\n",
    "  predictions = model.transform(test_data)\n",
    "  score = evaluator.evaluate(predictions) * 100\n",
    "\n",
    "  models[name] = (model, score, elapsed)\n",
    "  print(model, '\\nScore: %.2f [%5s ms]' % (score, elapsed))\n",
    "  return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "4874bf13-13b5-499b-ae05-a8d0b4d5e662",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# https://spark.apache.org/docs/latest/ml-tuning.html\n",
    "import numpy as np\n",
    "import re\n",
    "\n",
    "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
    "\n",
    "NUMBER_KFOLD_SPLITS = 5 # number of splits in cross-validation\n",
    "\n",
    "# faz o ajuste fino do modelo, calculando os melhores hiperparâmetros\n",
    "def fine_tune_model(estimator, params, evaluator, train_data, test_data):\n",
    "\n",
    "  print('\\nFine Tuning Model:')\n",
    "  print(estimator) #, \"\\nparams:\", params)\n",
    "  \n",
    "  crossval = CrossValidator(estimator=estimator,\n",
    "                            estimatorParamMaps=params,\n",
    "                            evaluator=evaluator,\n",
    "                            numFolds=NUMBER_KFOLD_SPLITS,\n",
    "                            seed=42,\n",
    "                            parallelism=3)\n",
    "  search = crossval.fit(train_data)\n",
    "\n",
    "  predictions = search.transform(test_data)\n",
    "  score = evaluator.evaluate(predictions) * 100\n",
    "\n",
    "  hyper_list = []\n",
    "  hyperparams = search.getEstimatorParamMaps()[np.argmax(search.avgMetrics)]\n",
    "  for i in range(len(hyperparams.items())):\n",
    "    hyper_name = re.search(\"name='(.+?)'\", str([x for x in hyperparams.items()][i])).group(1)\n",
    "    hyper_value = [x for x in hyperparams.items()][i][1]\n",
    "    hyper_list.append({hyper_name: hyper_value})\n",
    "\n",
    "  print('\\nBest Score: %.2f' % score)\n",
    "  print('Best Params:', hyper_list)\n",
    "  return search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "b48de5a8-95e6-428b-9b36-679dc8435c72",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[137]: &#39;from pyspark.ml.evaluation import MulticlassClassificationEvaluator\\n\\navaliador = MulticlassClassificationEvaluator(\\n    predictionCol = &#34;previsto&#34;,\\n    labelCol = &#34;real&#34;,\\n    metricName = &#34;accuracy&#34;\\n)&#39;</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[137]: &#39;from pyspark.ml.evaluation import MulticlassClassificationEvaluator\\n\\navaliador = MulticlassClassificationEvaluator(\\n    predictionCol = &#34;previsto&#34;,\\n    labelCol = &#34;real&#34;,\\n    metricName = &#34;accuracy&#34;\\n)&#39;</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "\n",
    "avaliador = MulticlassClassificationEvaluator(\n",
    "    predictionCol = \"previsto\",\n",
    "    labelCol = \"real\",\n",
    "    metricName = \"accuracy\"\n",
    ")'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "bea301e8-ef1c-4922-b0ad-7da09cc1c178",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "\n",
    "avaliador = BinaryClassificationEvaluator(\n",
    "  rawPredictionCol = \"previsto\",\n",
    "  labelCol = \"real\",\n",
    "  metricName = \"areaUnderROC\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "3a6ee2a4-fea9-4d7f-a254-b751c4c62106",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">LogisticRegressionModel: uid=LogisticRegression_e0ee0314a2a0, numClasses=2, numFeatures=16 \n",
       "Score: 94.95 [ 1193 ms]\n",
       "Out[139]: LogisticRegressionModel: uid=LogisticRegression_e0ee0314a2a0, numClasses=2, numFeatures=16</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">LogisticRegressionModel: uid=LogisticRegression_e0ee0314a2a0, numClasses=2, numFeatures=16 \nScore: 94.95 [ 1193 ms]\nOut[139]: LogisticRegressionModel: uid=LogisticRegression_e0ee0314a2a0, numClasses=2, numFeatures=16</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Logistic Regression\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "\n",
    "estimador = LogisticRegression(\n",
    "  regParam=0.1,\n",
    "  maxIter=10,\n",
    "  labelCol=\"real\",\n",
    "  featuresCol=\"features\",\n",
    "  predictionCol=\"previsto\"\n",
    ")\n",
    "\n",
    "params = ParamGridBuilder() \\\n",
    "  .addGrid(estimador.regParam, [1.0, 0.1, 0.01]) \\\n",
    "  .addGrid(estimador.maxIter, [10, 100, 1000]) \\\n",
    "  .build()\n",
    "#fine_tune_model(estimador, params, avaliador, dados_treino, dados_teste)\n",
    "\n",
    "evaluate_model('LR', estimador, avaliador, dados_treino, dados_teste)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "673793c7-2d32-425d-9259-b8c5833075e8",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">DecisionTreeClassificationModel: uid=DecisionTreeClassifier_85b02cbceb4d, depth=9, numNodes=199, numClasses=2, numFeatures=16 \n",
       "Score: 99.62 [ 1638 ms]\n",
       "Out[140]: DecisionTreeClassificationModel: uid=DecisionTreeClassifier_85b02cbceb4d, depth=9, numNodes=199, numClasses=2, numFeatures=16</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">DecisionTreeClassificationModel: uid=DecisionTreeClassifier_85b02cbceb4d, depth=9, numNodes=199, numClasses=2, numFeatures=16 \nScore: 99.62 [ 1638 ms]\nOut[140]: DecisionTreeClassificationModel: uid=DecisionTreeClassifier_85b02cbceb4d, depth=9, numNodes=199, numClasses=2, numFeatures=16</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Decision Tree\n",
    "from pyspark.ml.classification import DecisionTreeClassifier\n",
    "\n",
    "estimador = DecisionTreeClassifier(\n",
    "  maxDepth=9,\n",
    "  labelCol=\"real\",\n",
    "  featuresCol=\"features\",\n",
    "  predictionCol=\"previsto\"\n",
    ")\n",
    "\n",
    "params = ParamGridBuilder() \\\n",
    "  .addGrid(estimador.maxDepth, [2, 5, 7, 9, 11, 13]) \\\n",
    "  .build()\n",
    "#fine_tune_model(estimador, params, avaliador, dados_treino, dados_teste)\n",
    "\n",
    "evaluate_model('DT', estimador, avaliador, dados_treino, dados_teste)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "b3fe8ae8-c54b-47f7-9cf7-ef837cae9579",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">RandomForestClassificationModel: uid=RandomForestClassifier_a1de9875851e, numTrees=20, numClasses=2, numFeatures=16 \n",
       "Score: 99.19 [20563 ms]\n",
       "Out[141]: RandomForestClassificationModel: uid=RandomForestClassifier_a1de9875851e, numTrees=20, numClasses=2, numFeatures=16</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">RandomForestClassificationModel: uid=RandomForestClassifier_a1de9875851e, numTrees=20, numClasses=2, numFeatures=16 \nScore: 99.19 [20563 ms]\nOut[141]: RandomForestClassificationModel: uid=RandomForestClassifier_a1de9875851e, numTrees=20, numClasses=2, numFeatures=16</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Random Forest\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "\n",
    "estimador = RandomForestClassifier(\n",
    "  maxDepth=19,\n",
    "  labelCol=\"real\",\n",
    "  featuresCol=\"features\",\n",
    "  predictionCol=\"previsto\"\n",
    ")\n",
    "\n",
    "params = ParamGridBuilder() \\\n",
    "  .addGrid(estimador.maxDepth, [9, 11, 13, 17, 19]) \\\n",
    "  .build()\n",
    "#fine_tune_model(estimador, params, avaliador, dados_treino, dados_teste)\n",
    "\n",
    "evaluate_model('RF', estimador, avaliador, dados_treino, dados_teste)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "bfa9f3dc-c599-4db5-986d-de0edf9b5f31",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">GBTClassificationModel: uid = GBTClassifier_fde82d398d7a, numTrees=20, numClasses=2, numFeatures=16 \n",
       "Score: 98.93 [16126 ms]\n",
       "Out[142]: GBTClassificationModel: uid = GBTClassifier_fde82d398d7a, numTrees=20, numClasses=2, numFeatures=16</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">GBTClassificationModel: uid = GBTClassifier_fde82d398d7a, numTrees=20, numClasses=2, numFeatures=16 \nScore: 98.93 [16126 ms]\nOut[142]: GBTClassificationModel: uid = GBTClassifier_fde82d398d7a, numTrees=20, numClasses=2, numFeatures=16</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Gradient-Boosted Trees (GBTs)\n",
    "from pyspark.ml.classification import GBTClassifier\n",
    "\n",
    "estimador = GBTClassifier(\n",
    "  #maxDepth=10,\n",
    "  labelCol=\"real\",\n",
    "  featuresCol=\"features\",\n",
    "  predictionCol=\"previsto\"\n",
    ")\n",
    "\n",
    "params = ParamGridBuilder() \\\n",
    "  .addGrid(estimador.maxDepth, [5, 10]) \\\n",
    "  .build()\n",
    "#fine_tune_model(estimador, params, avaliador, dados_treino, dados_teste)\n",
    "\n",
    "# só classificação binária!\n",
    "evaluate_model('GB', estimador, avaliador, dados_treino, dados_teste)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "651ab842-808e-49b1-a323-8a65540668a5",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Multilayer Perceptron (MLP)\n",
    "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
    "\n",
    "estimador = MultilayerPerceptronClassifier(\n",
    "  #maxIter=100,\n",
    "  layers=[4, 5, 3],\n",
    "  #blockSize=128,\n",
    "  seed=42,\n",
    "  labelCol=\"real\",\n",
    "  featuresCol=\"features\",\n",
    "  predictionCol=\"previsto\"\n",
    ")\n",
    "\n",
    "#params = ParamGridBuilder() \\\n",
    "#  .addGrid(estimador.layers, [[4,5,3], [4,8,3], [4,5,6,3]]) \\\n",
    "#  .build()\n",
    "#fine_tune_model(estimador, params, avaliador, dados_treino, dados_teste)\n",
    "\n",
    "# está dando erro...\n",
    "#evaluate_model('MLP', estimador, avaliador, dados_treino, dados_teste)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "295690ad-e8c9-4a57-b70b-77f1adee3378",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">LinearSVCModel: uid=LinearSVC_ff1f9f0ae6b9, numClasses=2, numFeatures=16 \n",
       "Score: 95.28 [11954 ms]\n",
       "Out[144]: LinearSVCModel: uid=LinearSVC_ff1f9f0ae6b9, numClasses=2, numFeatures=16</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">LinearSVCModel: uid=LinearSVC_ff1f9f0ae6b9, numClasses=2, numFeatures=16 \nScore: 95.28 [11954 ms]\nOut[144]: LinearSVCModel: uid=LinearSVC_ff1f9f0ae6b9, numClasses=2, numFeatures=16</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Linear Support Vector Machine (SVM)\n",
    "from pyspark.ml.classification import LinearSVC\n",
    "\n",
    "estimador = LinearSVC(\n",
    "  maxIter=100,\n",
    "  regParam=0.1,\n",
    "  labelCol=\"real\",\n",
    "  featuresCol=\"features\",\n",
    "  predictionCol=\"previsto\"\n",
    ")\n",
    "\n",
    "params = ParamGridBuilder() \\\n",
    "  .addGrid(estimador.maxIter, [10, 50, 100, 500]) \\\n",
    "  .addGrid(estimador.regParam, [0.05, 0.1, 0.5]) \\\n",
    "  .build()\n",
    "#fine_tune_model(estimador, params, avaliador, dados_treino, dados_teste)\n",
    "\n",
    "# só classificação binária!\n",
    "evaluate_model('LSVM', estimador, avaliador, dados_treino, dados_teste)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "0a1ed32c-a3c7-4651-af68-5f70705dc3a5",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">NaiveBayesModel: uid=NaiveBayes_47013e5e1d76, modelType=multinomial, numClasses=2, numFeatures=16 \n",
       "Score: 91.98 [  893 ms]\n",
       "Out[145]: NaiveBayesModel: uid=NaiveBayes_47013e5e1d76, modelType=multinomial, numClasses=2, numFeatures=16</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">NaiveBayesModel: uid=NaiveBayes_47013e5e1d76, modelType=multinomial, numClasses=2, numFeatures=16 \nScore: 91.98 [  893 ms]\nOut[145]: NaiveBayesModel: uid=NaiveBayes_47013e5e1d76, modelType=multinomial, numClasses=2, numFeatures=16</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Naïve Bayes\n",
    "from pyspark.ml.classification import NaiveBayes\n",
    "\n",
    "estimador = NaiveBayes(\n",
    "  smoothing=0.25,\n",
    "  modelType=\"multinomial\",\n",
    "  labelCol=\"real\",\n",
    "  featuresCol=\"features\",\n",
    "  predictionCol=\"previsto\"\n",
    ")\n",
    "\n",
    "params = ParamGridBuilder() \\\n",
    "  .addGrid(estimador.smoothing, [0.0, 0.25, 0.5, 0.75, 1.0]) \\\n",
    "  .addGrid(estimador.modelType, [\"multinomial\"]) \\\n",
    "  .build()\n",
    "#fine_tune_model(estimador, params, avaliador, dados_treino, dados_teste)\n",
    "\n",
    "evaluate_model('NB', estimador, avaliador, dados_treino, dados_teste)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "4c5d6d0c-dc84-4789-86d9-e31bbced6ebc",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">FMClassificationModel: uid=FMClassifier_cacb780c46c1, numClasses=2, numFeatures=16, factorSize=8, fitLinear=true, fitIntercept=true \n",
       "Score: 95.03 [34626 ms]\n",
       "Out[146]: FMClassificationModel: uid=FMClassifier_cacb780c46c1, numClasses=2, numFeatures=16, factorSize=8, fitLinear=true, fitIntercept=true</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">FMClassificationModel: uid=FMClassifier_cacb780c46c1, numClasses=2, numFeatures=16, factorSize=8, fitLinear=true, fitIntercept=true \nScore: 95.03 [34626 ms]\nOut[146]: FMClassificationModel: uid=FMClassifier_cacb780c46c1, numClasses=2, numFeatures=16, factorSize=8, fitLinear=true, fitIntercept=true</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Factorization Machines (FM)\n",
    "from pyspark.ml.classification import FMClassifier\n",
    "\n",
    "estimador = FMClassifier(\n",
    "  stepSize=0.001,\n",
    "  labelCol=\"real\",\n",
    "  featuresCol=\"features\",\n",
    "  predictionCol=\"previsto\"\n",
    ")\n",
    "\n",
    "params = ParamGridBuilder() \\\n",
    "  .addGrid(estimador.stepSize, [0.001, 0.01, 0.1, 1.0]) \\\n",
    "  .build()\n",
    "#fine_tune_model(estimador, params, avaliador, dados_treino, dados_teste)\n",
    "\n",
    "# só classificação binária!\n",
    "evaluate_model('FM', estimador, avaliador, dados_treino, dados_teste)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "a2986f11-19d9-4dc9-947b-92e4eeeef713",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#results = []\n",
    "#for key, value in models.items():\n",
    "#  tup = (key,) + value\n",
    "#  results.append(tup)\n",
    "#results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "cb5b8347-9ab1-494e-ba69-7c8b3deae11e",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[148]: &#39;from pyspark.sql.types import StructType, StructField, StringType, IntegerType, FloatType\\nschema = StructType([   StructField(&#34;Model&#34;, StringType(), True),   StructField(&#34;Estimator&#34;, StringType(), True),   StructField(&#34;Score&#34;, FloatType(), True),   StructField(&#34;Time (ms)&#34;, IntegerType(), True) ])&#39;</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[148]: &#39;from pyspark.sql.types import StructType, StructField, StringType, IntegerType, FloatType\\nschema = StructType([   StructField(&#34;Model&#34;, StringType(), True),   StructField(&#34;Estimator&#34;, StringType(), True),   StructField(&#34;Score&#34;, FloatType(), True),   StructField(&#34;Time (ms)&#34;, IntegerType(), True) ])&#39;</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''from pyspark.sql.types import StructType, StructField, StringType, IntegerType, FloatType\n",
    "schema = StructType([ \\\n",
    "  StructField(\"Model\", StringType(), True), \\\n",
    "  StructField(\"Estimator\", StringType(), True), \\\n",
    "  StructField(\"Score\", FloatType(), True), \\\n",
    "  StructField(\"Time (ms)\", IntegerType(), True) \\\n",
    "])'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "95725d53-0156-420e-81d4-56a10ed88dd2",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[149]: &#39;results_df = spark.createDataFrame(data=results, schema=schema)\\nresults_df.printSchema()\\nresults_df.show(truncate=False)&#39;</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[149]: &#39;results_df = spark.createDataFrame(data=results, schema=schema)\\nresults_df.printSchema()\\nresults_df.show(truncate=False)&#39;</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''results_df = spark.createDataFrame(data=results, schema=schema)\n",
    "results_df.printSchema()\n",
    "results_df.show(truncate=False)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "3abd3426-5300-4fdd-9f13-9d209aa82827",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "names = []\n",
    "estimators = []\n",
    "scores = []\n",
    "times = []\n",
    "\n",
    "for key, value in models.items():\n",
    "  (model, score, elapsed) = value\n",
    "  names.append(key)\n",
    "  estimators.append(model)\n",
    "  scores.append(score)\n",
    "  times.append(elapsed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "66c07516-466e-4b7b-88ac-36b2af98c939",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[151]: </div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[151]: </div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time (ms)</th>\n",
       "      <th>Estimator</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DT</td>\n",
       "      <td>99.615126</td>\n",
       "      <td>1638</td>\n",
       "      <td>DecisionTreeClassificationModel: uid=DecisionT...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RF</td>\n",
       "      <td>99.190351</td>\n",
       "      <td>20563</td>\n",
       "      <td>RandomForestClassificationModel: uid=RandomFor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GB</td>\n",
       "      <td>98.934773</td>\n",
       "      <td>16126</td>\n",
       "      <td>GBTClassificationModel: uid = GBTClassifier_fd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LSVM</td>\n",
       "      <td>95.275978</td>\n",
       "      <td>11954</td>\n",
       "      <td>LinearSVCModel: uid=LinearSVC_ff1f9f0ae6b9, nu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>FM</td>\n",
       "      <td>95.027094</td>\n",
       "      <td>34626</td>\n",
       "      <td>FMClassificationModel: uid=FMClassifier_cacb78...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LR</td>\n",
       "      <td>94.949332</td>\n",
       "      <td>1193</td>\n",
       "      <td>LogisticRegressionModel: uid=LogisticRegressio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NB</td>\n",
       "      <td>91.981002</td>\n",
       "      <td>893</td>\n",
       "      <td>NaiveBayesModel: uid=NaiveBayes_47013e5e1d76, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Model</th>\n      <th>Score</th>\n      <th>Time (ms)</th>\n      <th>Estimator</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>DT</td>\n      <td>99.615126</td>\n      <td>1638</td>\n      <td>DecisionTreeClassificationModel: uid=DecisionT...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>RF</td>\n      <td>99.190351</td>\n      <td>20563</td>\n      <td>RandomForestClassificationModel: uid=RandomFor...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>GB</td>\n      <td>98.934773</td>\n      <td>16126</td>\n      <td>GBTClassificationModel: uid = GBTClassifier_fd...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>LSVM</td>\n      <td>95.275978</td>\n      <td>11954</td>\n      <td>LinearSVCModel: uid=LinearSVC_ff1f9f0ae6b9, nu...</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>FM</td>\n      <td>95.027094</td>\n      <td>34626</td>\n      <td>FMClassificationModel: uid=FMClassifier_cacb78...</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>LR</td>\n      <td>94.949332</td>\n      <td>1193</td>\n      <td>LogisticRegressionModel: uid=LogisticRegressio...</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>NB</td>\n      <td>91.981002</td>\n      <td>893</td>\n      <td>NaiveBayesModel: uid=NaiveBayes_47013e5e1d76, ...</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "textData": null,
       "type": "htmlSandbox"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "results_df = pd.DataFrame({\n",
    "  'Model': names,\n",
    "  'Score': scores,\n",
    "  'Time (ms)': times,\n",
    "  'Estimator': estimators})\n",
    "\n",
    "results_df.sort_values(by='Score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "2774e250-cc60-4824-8cad-dc94031e423e",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[152]: DecisionTreeClassificationModel: uid=DecisionTreeClassifier_85b02cbceb4d, depth=9, numNodes=199, numClasses=2, numFeatures=16</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[152]: DecisionTreeClassificationModel: uid=DecisionTreeClassifier_85b02cbceb4d, depth=9, numNodes=199, numClasses=2, numFeatures=16</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "modelo = models['DT'][0]\n",
    "modelo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "c8188d8e-8664-4051-a7a9-d30499905c0a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "arguments": {},
       "data": "",
       "errorSummary": "",
       "type": "ipynbError"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "70521f24-7cb2-4b73-80a3-c20020531d3d",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[128]: RandomForestClassifier_ebbb295d0351</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[128]: RandomForestClassifier_ebbb295d0351</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dados_treino = df5 # treinar com todas as linhas\n",
    "\n",
    "estimador = RandomForestClassifier(\n",
    "  maxDepth=19,\n",
    "  labelCol=\"real\",\n",
    "  featuresCol=\"features\",\n",
    "  predictionCol=\"previsto\"\n",
    ")\n",
    "\n",
    "estimador.fit(dados_treino)\n",
    "estimador"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "1a3f3041-7f27-407a-9ea6-f3f1c779f0fc",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# carregar dados de teste\n",
    "# ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "19b332d9-a24c-4355-a314-1488fed372a1",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dados_teste = df5 # usar todas as linhas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "d81a7832-c7f5-4677-aab5-77be5d750358",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "arguments": {},
       "data": "",
       "errorSummary": "",
       "type": "ipynbError"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "9ca66056-8792-4dcd-8d90-b8decee1e6fb",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+------------+----+--------+\n",
       "        nome|real|previsto|\n",
       "+------------+----+--------+\n",
       "    AARONSON|   1|     1.0|\n",
       "      ABADIO|   1|     1.0|\n",
       "      ABAITE|   0|     0.0|\n",
       "        ABDA|   0|     0.0|\n",
       "    ABDALLAH|   0|     0.0|\n",
       "   ABDEILDES|   1|     1.0|\n",
       "ABDELMOUBINE|   0|     0.0|\n",
       "      ABDIAS|   1|     1.0|\n",
       "   ABDINARDO|   1|     1.0|\n",
       "    ABDNAGOS|   1|     1.0|\n",
       "+------------+----+--------+\n",
       "only showing top 10 rows\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">+------------+----+--------+\n|        nome|real|previsto|\n+------------+----+--------+\n|    AARONSON|   1|     1.0|\n|      ABADIO|   1|     1.0|\n|      ABAITE|   0|     0.0|\n|        ABDA|   0|     0.0|\n|    ABDALLAH|   0|     0.0|\n|   ABDEILDES|   1|     1.0|\n|ABDELMOUBINE|   0|     0.0|\n|      ABDIAS|   1|     1.0|\n|   ABDINARDO|   1|     1.0|\n|    ABDNAGOS|   1|     1.0|\n+------------+----+--------+\nonly showing top 10 rows\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# gerar previsões e mostrar exemplos\n",
    "previsoes = modelo.transform(dados_teste)\n",
    "previsoes.select(\"nome\", \"real\", \"previsto\").show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "738a95c5-f9eb-4081-a266-b4b1c270d183",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[154]: 99.61512646494813</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[154]: 99.61512646494813</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "score = avaliador.evaluate(previsoes) * 100\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "340e0a62-1c86-497c-942e-e70267d2ed94",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>nome</th><th>real</th><th>features</th><th>rawPrediction</th><th>probability</th><th>previsto</th></tr></thead><tbody><tr><td>AARONSON</td><td>1</td><td>Map(vectorType -> dense, length -> 16, values -> List(14.0, 15.0, 19.0, 14.0, 15.0, 18.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.0, 2236.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.0, 1.0))</td><td>1.0</td></tr><tr><td>ABADIO</td><td>1</td><td>Map(vectorType -> dense, length -> 16, values -> List(15.0, 9.0, 4.0, 1.0, 2.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.0, 5894.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.6963528413910093E-4, 0.999830364715861))</td><td>1.0</td></tr><tr><td>ABAITE</td><td>0</td><td>Map(vectorType -> dense, length -> 16, values -> List(5.0, 20.0, 9.0, 1.0, 2.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(746.0, 8.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.9893899204244032, 0.010610079575596816))</td><td>0.0</td></tr><tr><td>ABDA</td><td>0</td><td>Map(vectorType -> dense, length -> 16, values -> List(1.0, 4.0, 2.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(7789.0, 0.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.0, 0.0))</td><td>0.0</td></tr><tr><td>ABDALLAH</td><td>0</td><td>Map(vectorType -> dense, length -> 16, values -> List(8.0, 1.0, 12.0, 12.0, 1.0, 4.0, 2.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(72.0, 0.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.0, 0.0))</td><td>0.0</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         "AARONSON",
         1,
         {
          "length": 16,
          "values": [
           14,
           15,
           19,
           14,
           15,
           18,
           1,
           1,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0,
           2236
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0,
           1
          ],
          "vectorType": "dense"
         },
         1
        ],
        [
         "ABADIO",
         1,
         {
          "length": 16,
          "values": [
           15,
           9,
           4,
           1,
           2,
           1,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1,
           5894
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.00016963528413910093,
           0.999830364715861
          ],
          "vectorType": "dense"
         },
         1
        ],
        [
         "ABAITE",
         0,
         {
          "length": 16,
          "values": [
           5,
           20,
           9,
           1,
           2,
           1,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           746,
           8
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.9893899204244032,
           0.010610079575596816
          ],
          "vectorType": "dense"
         },
         0
        ],
        [
         "ABDA",
         0,
         {
          "length": 16,
          "values": [
           1,
           4,
           2,
           1,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           7789,
           0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1,
           0
          ],
          "vectorType": "dense"
         },
         0
        ],
        [
         "ABDALLAH",
         0,
         {
          "length": 16,
          "values": [
           8,
           1,
           12,
           12,
           1,
           4,
           2,
           1,
           0,
           0,
           0,
           0,
           0,
           0,
           0,
           0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           72,
           0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1,
           0
          ],
          "vectorType": "dense"
         },
         0
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "nome",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "real",
         "type": "\"long\""
        },
        {
         "metadata": "{}",
         "name": "features",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{}",
         "name": "rawPrediction",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{}",
         "name": "probability",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{}",
         "name": "previsto",
         "type": "\"double\""
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(previsoes.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "fb29be9d-64c3-43db-95bb-898a452b92d0",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+----+--------+-----+\n",
       "real|previsto|count|\n",
       "+----+--------+-----+\n",
       "   1|     0.0|   23|\n",
       "   0|     1.0|   24|\n",
       "   0|     0.0| 6538|\n",
       "   1|     1.0| 5670|\n",
       "+----+--------+-----+\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">+----+--------+-----+\n|real|previsto|count|\n+----+--------+-----+\n|   1|     0.0|   23|\n|   0|     1.0|   24|\n|   0|     0.0| 6538|\n|   1|     1.0| 5670|\n+----+--------+-----+\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# matriz de confusão\n",
    "previsoes.groupBy(\"real\", \"previsto\").count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "1add0c85-a72b-4ab7-937a-40da67b70e42",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+-------+----+--------+\n",
       "   nome|real|previsto|\n",
       "+-------+----+--------+\n",
       "   ANNY|   0|     1.0|\n",
       "  DORIS|   0|     1.0|\n",
       " ELENIR|   0|     1.0|\n",
       " GLADIS|   0|     1.0|\n",
       " GLEDSE|   0|     1.0|\n",
       " IASMIM|   0|     1.0|\n",
       " ISABEL|   0|     1.0|\n",
       "   LADY|   0|     1.0|\n",
       " LEONOR|   0|     1.0|\n",
       "LILLIAN|   0|     1.0|\n",
       "+-------+----+--------+\n",
       "only showing top 10 rows\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">+-------+----+--------+\n|   nome|real|previsto|\n+-------+----+--------+\n|   ANNY|   0|     1.0|\n|  DORIS|   0|     1.0|\n| ELENIR|   0|     1.0|\n| GLADIS|   0|     1.0|\n| GLEDSE|   0|     1.0|\n| IASMIM|   0|     1.0|\n| ISABEL|   0|     1.0|\n|   LADY|   0|     1.0|\n| LEONOR|   0|     1.0|\n|LILLIAN|   0|     1.0|\n+-------+----+--------+\nonly showing top 10 rows\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "previsoes.filter(\"real = 0\").filter(\"real != previsto\").select(\"nome\", \"real\", \"previsto\").show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "98351ffd-5768-46f8-9705-987635ee6f42",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+-------+----+--------+\n",
       "   nome|real|previsto|\n",
       "+-------+----+--------+\n",
       " ADONAI|   1|     0.0|\n",
       "   ALIX|   1|     0.0|\n",
       "  BRUCE|   1|     0.0|\n",
       " DESAIX|   1|     0.0|\n",
       "    ELI|   1|     0.0|\n",
       "    ELY|   1|     0.0|\n",
       " ESAULL|   1|     0.0|\n",
       " GEORGE|   1|     0.0|\n",
       "  HERLY|   1|     0.0|\n",
       "HERNANI|   1|     0.0|\n",
       "+-------+----+--------+\n",
       "only showing top 10 rows\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">+-------+----+--------+\n|   nome|real|previsto|\n+-------+----+--------+\n| ADONAI|   1|     0.0|\n|   ALIX|   1|     0.0|\n|  BRUCE|   1|     0.0|\n| DESAIX|   1|     0.0|\n|    ELI|   1|     0.0|\n|    ELY|   1|     0.0|\n| ESAULL|   1|     0.0|\n| GEORGE|   1|     0.0|\n|  HERLY|   1|     0.0|\n|HERNANI|   1|     0.0|\n+-------+----+--------+\nonly showing top 10 rows\n\n</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "previsoes.filter(\"real = 1\").filter(\"real != previsto\").select(\"nome\", \"real\", \"previsto\").show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "76b807c2-a225-4eb2-b51f-3c23a7acccc3",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "fea9175f-0ac4-4094-98ca-acfe6fe57de8",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# https://docs.databricks.com/applications/machine-learning/model-export/mleap-model-export.html#export-and-import-models-in-python\n",
    "#!pip install mleap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "c3463470-0502-4ec9-9219-bfe904853adb",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''%sh\n",
    "rm -rf /tmp/mleap_python_model_export\n",
    "mkdir /tmp/mleap_python_model_export'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "9054d370-1c14-423c-9500-e3843a11439d",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"><span class=\"ansi-red-fg\">---------------------------------------------------------------------------</span>\n",
       "<span class=\"ansi-red-fg\">TypeError</span>                                 Traceback (most recent call last)\n",
       "<span class=\"ansi-green-fg\">&lt;command-723065229944245&gt;</span> in <span class=\"ansi-cyan-fg\">&lt;module&gt;</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">      2</span> <span class=\"ansi-green-fg\">from</span> mleap<span class=\"ansi-blue-fg\">.</span>pyspark<span class=\"ansi-blue-fg\">.</span>spark_support <span class=\"ansi-green-fg\">import</span> SimpleSparkSerializer\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">      3</span> \n",
       "<span class=\"ansi-green-fg\">----&gt; 4</span><span class=\"ansi-red-fg\"> </span>modelo<span class=\"ansi-blue-fg\">.</span>serializeToBundle<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;jar:file:/tmp/mleap_python_model_export/72-modelo-genero-nome-pyspark-json.zip&#34;</span><span class=\"ansi-blue-fg\">,</span> previsoes<span class=\"ansi-blue-fg\">)</span>\n",
       "\n",
       "<span class=\"ansi-green-fg\">/databricks/python/lib/python3.8/site-packages/mleap/pyspark/spark_support.py</span> in <span class=\"ansi-cyan-fg\">serializeToBundle</span><span class=\"ansi-blue-fg\">(self, path, dataset)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">     22</span> \n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">     23</span> <span class=\"ansi-green-fg\">def</span> serializeToBundle<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> path<span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">=</span><span class=\"ansi-green-fg\">None</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n",
       "<span class=\"ansi-green-fg\">---&gt; 24</span><span class=\"ansi-red-fg\">     </span>serializer <span class=\"ansi-blue-fg\">=</span> SimpleSparkSerializer<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">     25</span>     serializer<span class=\"ansi-blue-fg\">.</span>serializeToBundle<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> path<span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">=</span>dataset<span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">     26</span> \n",
       "\n",
       "<span class=\"ansi-green-fg\">/databricks/python/lib/python3.8/site-packages/mleap/pyspark/spark_support.py</span> in <span class=\"ansi-cyan-fg\">__init__</span><span class=\"ansi-blue-fg\">(self)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">     37</span>     <span class=\"ansi-green-fg\">def</span> __init__<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">     38</span>         super<span class=\"ansi-blue-fg\">(</span>SimpleSparkSerializer<span class=\"ansi-blue-fg\">,</span> self<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>__init__<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-fg\">---&gt; 39</span><span class=\"ansi-red-fg\">         </span>self<span class=\"ansi-blue-fg\">.</span>_java_obj <span class=\"ansi-blue-fg\">=</span> _jvm<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>ml<span class=\"ansi-blue-fg\">.</span>combust<span class=\"ansi-blue-fg\">.</span>mleap<span class=\"ansi-blue-fg\">.</span>spark<span class=\"ansi-blue-fg\">.</span>SimpleSparkSerializer<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">     40</span> \n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">     41</span>     <span class=\"ansi-green-fg\">def</span> serializeToBundle<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> transformer<span class=\"ansi-blue-fg\">,</span> path<span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n",
       "\n",
       "<span class=\"ansi-red-fg\">TypeError</span>: &#39;JavaPackage&#39; object is not callable</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "arguments": {},
       "data": "<div class=\"ansiout\"><span class=\"ansi-red-fg\">---------------------------------------------------------------------------</span>\n<span class=\"ansi-red-fg\">TypeError</span>                                 Traceback (most recent call last)\n<span class=\"ansi-green-fg\">&lt;command-723065229944245&gt;</span> in <span class=\"ansi-cyan-fg\">&lt;module&gt;</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">      2</span> <span class=\"ansi-green-fg\">from</span> mleap<span class=\"ansi-blue-fg\">.</span>pyspark<span class=\"ansi-blue-fg\">.</span>spark_support <span class=\"ansi-green-fg\">import</span> SimpleSparkSerializer\n<span class=\"ansi-green-intense-fg ansi-bold\">      3</span> \n<span class=\"ansi-green-fg\">----&gt; 4</span><span class=\"ansi-red-fg\"> </span>modelo<span class=\"ansi-blue-fg\">.</span>serializeToBundle<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;jar:file:/tmp/mleap_python_model_export/72-modelo-genero-nome-pyspark-json.zip&#34;</span><span class=\"ansi-blue-fg\">,</span> previsoes<span class=\"ansi-blue-fg\">)</span>\n\n<span class=\"ansi-green-fg\">/databricks/python/lib/python3.8/site-packages/mleap/pyspark/spark_support.py</span> in <span class=\"ansi-cyan-fg\">serializeToBundle</span><span class=\"ansi-blue-fg\">(self, path, dataset)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">     22</span> \n<span class=\"ansi-green-intense-fg ansi-bold\">     23</span> <span class=\"ansi-green-fg\">def</span> serializeToBundle<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> path<span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">=</span><span class=\"ansi-green-fg\">None</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-fg\">---&gt; 24</span><span class=\"ansi-red-fg\">     </span>serializer <span class=\"ansi-blue-fg\">=</span> SimpleSparkSerializer<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">     25</span>     serializer<span class=\"ansi-blue-fg\">.</span>serializeToBundle<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> path<span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">=</span>dataset<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">     26</span> \n\n<span class=\"ansi-green-fg\">/databricks/python/lib/python3.8/site-packages/mleap/pyspark/spark_support.py</span> in <span class=\"ansi-cyan-fg\">__init__</span><span class=\"ansi-blue-fg\">(self)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">     37</span>     <span class=\"ansi-green-fg\">def</span> __init__<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">     38</span>         super<span class=\"ansi-blue-fg\">(</span>SimpleSparkSerializer<span class=\"ansi-blue-fg\">,</span> self<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>__init__<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-fg\">---&gt; 39</span><span class=\"ansi-red-fg\">         </span>self<span class=\"ansi-blue-fg\">.</span>_java_obj <span class=\"ansi-blue-fg\">=</span> _jvm<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>ml<span class=\"ansi-blue-fg\">.</span>combust<span class=\"ansi-blue-fg\">.</span>mleap<span class=\"ansi-blue-fg\">.</span>spark<span class=\"ansi-blue-fg\">.</span>SimpleSparkSerializer<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">     40</span> \n<span class=\"ansi-green-intense-fg ansi-bold\">     41</span>     <span class=\"ansi-green-fg\">def</span> serializeToBundle<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> transformer<span class=\"ansi-blue-fg\">,</span> path<span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n\n<span class=\"ansi-red-fg\">TypeError</span>: &#39;JavaPackage&#39; object is not callable</div>",
       "errorSummary": "<span class=\"ansi-red-fg\">TypeError</span>: &#39;JavaPackage&#39; object is not callable",
       "type": "ipynbError"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''import mleap.pyspark\n",
    "from mleap.pyspark.spark_support import SimpleSparkSerializer\n",
    "\n",
    "modelo.serializeToBundle(\"jar:file:/tmp/mleap_python_model_export/72-modelo-genero-nome-pyspark-json.zip\", previsoes)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "422ae992-1061-42d5-b7a2-009a80a4c739",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''modelo.save('/tmp/72-modelo-genero-nome-pyspark-json')'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "3af2e16b-d9b6-4d0d-aacc-74813ddc5130",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[161]: DecisionTreeClassificationModel: uid=DecisionTreeClassifier_85b02cbceb4d, depth=9, numNodes=199, numClasses=2, numFeatures=16</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[161]: DecisionTreeClassificationModel: uid=DecisionTreeClassifier_85b02cbceb4d, depth=9, numNodes=199, numClasses=2, numFeatures=16</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "modelo2 = modelo.load('/tmp/72-modelo-genero-nome-pyspark-json')\n",
    "modelo2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "7cdb7ec1-0eb0-4183-968d-9e1a5c99949e",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[162]: 99.61512646494813</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[162]: 99.61512646494813</div>",
       "datasetInfos": [],
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "previsoes2 = modelo2.transform(dados_teste)\n",
    "avaliador.evaluate(previsoes2) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "aff55d4b-58eb-4481-b3b2-b5627213f06c",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "arguments": {},
       "data": "",
       "errorSummary": "",
       "type": "ipynbError"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "1e10cb77-76eb-413e-8f01-e082a16df114",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "dados = pd.DataFrame({'REAL': y, 'PREV': y_pred}, index=X.index)\n",
    "for col in dados.columns:\n",
    "    dados[col] = dados[col].map({0: 'X', 1: 'F', 2: 'M'})\n",
    "dados.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "a62b636b-bb04-4bef-a68a-500741875593",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracy_score(y, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "cab77263-acea-486f-9266-a33dbf9b9d36",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "597677a2-152d-467e-8190-32a25638c5e4",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "dados[dados['REAL'] != dados['PREV']].head(20)"
   ]
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 2
   },
   "notebookName": "72-modelo-genero-nome-pyspark",
   "notebookOrigID": 1866394035798127,
   "widgets": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}