colab+automl_bigquery.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "colab+automl-bigquery.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/noahgift/gcp-hello-ml/blob/master/colab%2Bautoml_bigquery.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5gP6_8YmxJul",
        "colab_type": "text"
      },
      "source": [
        "# Week 7 - USF"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rtHVEUlZxCbY",
        "colab_type": "text"
      },
      "source": [
        "## Intro to Colab"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X9rafdLrxJDI",
        "colab_type": "text"
      },
      "source": [
        "### Command-line"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m-I7LMV9xs3b",
        "colab_type": "text"
      },
      "source": [
        "#### execute commands"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9lvvNeeHwoLv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "a78a0301-7625-43bd-92f0-aa111c533ba4"
      },
      "source": [
        "!uname -a"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Linux 8c54306d4f3b 4.14.79+ #1 SMP Wed Dec 19 21:19:13 PST 2018 x86_64 x86_64 x86_64 GNU/Linux\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H21rbrrIxwmA",
        "colab_type": "text"
      },
      "source": [
        "#### capture commands"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jgQD3R1ExalW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "90775c28-f967-4f8c-f4df-27352d0d082e"
      },
      "source": [
        "var = !ls -l\n",
        "type(var)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "IPython.utils.text.SList"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zuq587vYxjsy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "598a4d1e-eb67-445e-973a-de224d272928"
      },
      "source": [
        "var.grep(\"data\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['drwxr-xr-x 1 root root 4096 Jun 18 16:14 sample_data']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AFPc5O-9ydPl",
        "colab_type": "text"
      },
      "source": [
        "#### Install Software inside colab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gQCoqNgayh4D",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "8a9c64e1-27af-4d82-87bb-cdf24da61e6a"
      },
      "source": [
        "!pip install -q shap"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[?25l\r\u001b[K     |█▍                              | 10kB 15.6MB/s eta 0:00:01\r\u001b[K     |██▉                             | 20kB 3.5MB/s eta 0:00:01\r\u001b[K     |████▎                           | 30kB 5.0MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 40kB 3.1MB/s eta 0:00:01\r\u001b[K     |███████                         | 51kB 3.8MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 61kB 4.5MB/s eta 0:00:01\r\u001b[K     |██████████                      | 71kB 5.2MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 81kB 5.9MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 92kB 6.5MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 102kB 5.0MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 112kB 5.0MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 122kB 5.0MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 133kB 5.0MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 143kB 5.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 153kB 5.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 163kB 5.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 174kB 5.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 184kB 5.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 194kB 5.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 204kB 5.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 215kB 5.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 225kB 5.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 235kB 5.0MB/s \n",
            "\u001b[?25h  Building wheel for shap (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HQlJ99QNyqgo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import shap"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yM5zWhiKx1JE",
        "colab_type": "text"
      },
      "source": [
        "### Cell Magic"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JxmBurKwyBTf",
        "colab_type": "text"
      },
      "source": [
        "##### Run Python2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w8HBn98bxoaN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "40bae72f-f37b-4287-c008-94f53e5e4879"
      },
      "source": [
        "%%python2\n",
        "\n",
        "print \"old school\""
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "old school\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "avoJlJzUySoU",
        "colab_type": "text"
      },
      "source": [
        "##### Run Bash"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LTFDtqf0x5om",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "433072e9-f628-47c1-d872-8465751ad37e"
      },
      "source": [
        "%%bash\n",
        "ls -l\n",
        "uname -a\n",
        "echo \"I am in bash\""
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 4\n",
            "drwxr-xr-x 1 root root 4096 Jun 18 16:14 sample_data\n",
            "Linux 8c54306d4f3b 4.14.79+ #1 SMP Wed Dec 19 21:19:13 PST 2018 x86_64 x86_64 x86_64 GNU/Linux\n",
            "I am in bash\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HaTbN1aByZmn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mx9MyMQnyvam",
        "colab_type": "text"
      },
      "source": [
        "##### Use writefile + pylint"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_KVDqXkDy_Kb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "0f36fb4f-e82f-42a2-941f-14ef31bf0dea"
      },
      "source": [
        "!pip install -q pylint"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 768kB 4.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 18.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 194kB 44.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 737kB 45.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 19.4MB/s \n",
            "\u001b[?25h"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-IRLQztSy1M6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c6fbe901-0df6-44a9-e74a-5b6af3014826"
      },
      "source": [
        "%%writefile out.py\n",
        "print(\"good code\")\n",
        "print(badcode)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Writing out.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r37ZzSXwy8oS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "outputId": "1e8b0dac-cc37-4bf0-afac-5aee834f8c84"
      },
      "source": [
        "!pylint out.py"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "************* Module out\n",
            "out.py:2:0: C0304: Final newline missing (missing-final-newline)\n",
            "out.py:1:0: C0111: Missing module docstring (missing-docstring)\n",
            "out.py:2:6: E0602: Undefined variable 'badcode' (undefined-variable)\n",
            "\n",
            "-------------------------------------\n",
            "Your code has been rated at -25.00/10\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ZA3GCLqzFLf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YfNVSi49zWGw",
        "colab_type": "text"
      },
      "source": [
        "### Files"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ez4eXiHX1WAy",
        "colab_type": "text"
      },
      "source": [
        "Mount GDrive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MmWpYyH8zvbX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "86a08acf-e0dd-407e-f7ab-2b8d9631a179"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive', force_remount=True)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LecNSQdM1YCl",
        "colab_type": "text"
      },
      "source": [
        "Verify the contents exist in bash and python"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pBskSVTaz-bo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "17ad4c8a-c08e-4ca5-f37e-c492504e9827"
      },
      "source": [
        "!ls -l /gdrive/My\\ Drive/colab_data"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 4\n",
            "-rw------- 1 root root 3536 Jul  3 21:51 education_sugar_cdc_2003.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PeD5mR180mLb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "de03ef94-a635-4be8-aec4-0a529f84dc2c"
      },
      "source": [
        "path = \"/gdrive/My Drive/colab_data/education_sugar_cdc_2003.csv\"\n",
        "import os;os.path.exists(path)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n8Sac_Z41TAD",
        "colab_type": "text"
      },
      "source": [
        "Use pandas to import"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "grddyT7S0_RZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        },
        "outputId": "e9aa5446-bd03-41f1-8672-04589266005c"
      },
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv(path)\n",
        "df.head()"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>State</th>\n",
              "      <th>Employed</th>\n",
              "      <th>Not employed</th>\n",
              "      <th>Retired</th>\n",
              "      <th>&lt;High school</th>\n",
              "      <th>High school</th>\n",
              "      <th>Some college</th>\n",
              "      <th>College graduate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Alaska</td>\n",
              "      <td>26.2 (23.6–28.9)</td>\n",
              "      <td>32.1 (27.8–36.8)</td>\n",
              "      <td>16.0 (12.6–20.2)</td>\n",
              "      <td>47.1 (37.8–56.5)</td>\n",
              "      <td>34.9 (31.1–38.9)</td>\n",
              "      <td>24.2 (21.0–27.8)</td>\n",
              "      <td>12.9 (10.5–15.7)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Arizona</td>\n",
              "      <td>33.0 (28.5–37.8)</td>\n",
              "      <td>28.7 (23.5–34.5)</td>\n",
              "      <td>13.8 (10.8–17.5)</td>\n",
              "      <td>40.4 (30.9–50.7)</td>\n",
              "      <td>36.5 (30.7–42.7)</td>\n",
              "      <td>24.4 (19.9–29.4)</td>\n",
              "      <td>14.6 (11.6–18.3)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>California</td>\n",
              "      <td>22.9 (20.9–25.1)</td>\n",
              "      <td>30.2 (27.1–33.4)</td>\n",
              "      <td>15.0 (12.2–18.2)</td>\n",
              "      <td>38.5 (34.2–43.0)</td>\n",
              "      <td>29.9 (26.5–33.7)</td>\n",
              "      <td>21.4 (18.8–24.2)</td>\n",
              "      <td>11.5 (9.8–13.5)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Connecticut</td>\n",
              "      <td>18.9 (17.1–20.9)</td>\n",
              "      <td>24.3 (20.8–28.2)</td>\n",
              "      <td>15.0 (12.7–17.7)</td>\n",
              "      <td>27.8 (22.4–33.9)</td>\n",
              "      <td>26.9 (23.7–30.3)</td>\n",
              "      <td>19.9 (17.2–23.0)</td>\n",
              "      <td>10.2 (8.7–12.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>District of Columbia</td>\n",
              "      <td>18.5 (15.7–21.7)</td>\n",
              "      <td>34.6 (29.5–40.1)</td>\n",
              "      <td>18.5 (15.3–22.1)</td>\n",
              "      <td>45.6 (36.4–55.2)</td>\n",
              "      <td>39.0 (33.1–45.2)</td>\n",
              "      <td>28.9 (23.4–35.0)</td>\n",
              "      <td>8.4 (7.0–10.1)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                  State          Employed  ...      Some college  College graduate\n",
              "0                Alaska  26.2 (23.6–28.9)  ...  24.2 (21.0–27.8)  12.9 (10.5–15.7)\n",
              "1               Arizona  33.0 (28.5–37.8)  ...  24.4 (19.9–29.4)  14.6 (11.6–18.3)\n",
              "2            California  22.9 (20.9–25.1)  ...  21.4 (18.8–24.2)   11.5 (9.8–13.5)\n",
              "3           Connecticut  18.9 (17.1–20.9)  ...  19.9 (17.2–23.0)   10.2 (8.7–12.0)\n",
              "4  District of Columbia  18.5 (15.7–21.7)  ...  28.9 (23.4–35.0)    8.4 (7.0–10.1)\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CW31_yuE1geL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}