Lit2Vec2TrainingPublic.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lit2Vec2TrainingPublic.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Santosh-Gupta/Lit2Vec2/blob/master/Lit2Vec2TrainingPublic.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V62piRJV2azh",
        "colab_type": "code",
        "outputId": "bdce14e4-f135-4e89-d118-626099cf2679",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!pip install LogUniform"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: LogUniform in /usr/local/lib/python3.6/dist-packages (1.0.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cz8FsZRH99zd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from torch.autograd import Variable\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "import re\n",
        "from collections import Counter\n",
        "import time\n",
        "\n",
        "import loguniform\n",
        "from loguniform import LogUniform\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "from __future__ import print_function\n",
        "import collections\n",
        "import math\n",
        "import numpy as np\n",
        "import random\n",
        "import zipfile\n",
        "import os\n",
        "import requests\n",
        "\n",
        "from urllib.request import urlretrieve\n",
        "from os.path import isfile, isdir\n",
        "import zipfile\n",
        "import tensorflow as tf\n",
        "from matplotlib import pylab\n",
        "from six.moves import range\n",
        "from six.moves.urllib.request import urlretrieve\n",
        "from sklearn.manifold import TSNE\n",
        "import pandas as pd\n",
        "\n",
        "from numpy import genfromtxt\n",
        "import sys\n",
        "\n",
        "numbBooks = 2829853+1 #according to https://github.com/zygmuntz/goodbooks-10k there are 10000 books in the dataset\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g7pnzPPq0okb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def download_file_from_google_drive(id, destination):\n",
        "    URL = \"https://docs.google.com/uc?export=download\"\n",
        "\n",
        "    session = requests.Session()\n",
        "\n",
        "    response = session.get(URL, params = { 'id' : id }, stream = True)\n",
        "    token = get_confirm_token(response)\n",
        "\n",
        "    if token:\n",
        "        params = { 'id' : id, 'confirm' : token }\n",
        "        response = session.get(URL, params = params, stream = True)\n",
        "\n",
        "    save_response_content(response, destination)    \n",
        "\n",
        "def get_confirm_token(response):\n",
        "    for key, value in response.cookies.items():\n",
        "        if key.startswith('download_warning'):\n",
        "            return value\n",
        "\n",
        "    return None\n",
        "\n",
        "def save_response_content(response, destination):\n",
        "    CHUNK_SIZE = 32768\n",
        "\n",
        "    with open(destination, \"wb\") as f:\n",
        "        for chunk in response.iter_content(CHUNK_SIZE):\n",
        "            if chunk: # filter out keep-alive new chunks\n",
        "                f.write(chunk)\n",
        "                "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fub14Dc60_kJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# download_file_from_google_drive('1w3uSNtc1srNaWoRHwbUE9o3baokloh9U', 'books.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ChkL1wlL4s3d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# dl_id = input(\"Enter Gdrive file ID for books: \") \n",
        "\n",
        "# dl_id =  '1w3uSNtc1srNaWoRHwbUE9o3baokloh9U'\n",
        "\n",
        "# # DOWNLOAD ZIP\n",
        "# print (\"Downloading  file\")\n",
        "# myzip = drive.CreateFile({'id': dl_id})\n",
        "# myzip.GetContentFile('books.csv')\n",
        "\n",
        "# print( os.listdir() )\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xkAhaGQZ4uOX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# b = pd.read_csv( 'books.csv' )\n",
        "# b.head(30)\n",
        "# bookDictionary = b.set_index('book_id').to_dict()['original_title']\n",
        "# bookDictionary[5]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jkf7DrmN-Jmv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "download_file_from_google_drive('1lH-0JqQ4UwgL-k83w9xrfftHzvx54phy', 'GoodReadsUser4MS.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yn_r-hAt8JP4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "openHdf = pd.read_hdf('GoodReadsUser4MS.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3dDUtTm18ben",
        "colab_type": "code",
        "outputId": "6c13fdd7-bdf3-4a13-81ea-b8a447091354",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "openHdf.loc[180:220]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UserID</th>\n",
              "      <th>EmbedID</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>180</th>\n",
              "      <td>1</td>\n",
              "      <td>180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>181</th>\n",
              "      <td>1</td>\n",
              "      <td>181</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>182</th>\n",
              "      <td>1</td>\n",
              "      <td>182</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>183</th>\n",
              "      <td>1</td>\n",
              "      <td>183</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>184</th>\n",
              "      <td>1</td>\n",
              "      <td>184</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>185</th>\n",
              "      <td>1</td>\n",
              "      <td>185</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>186</th>\n",
              "      <td>1</td>\n",
              "      <td>186</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>187</th>\n",
              "      <td>1</td>\n",
              "      <td>187</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>188</th>\n",
              "      <td>1</td>\n",
              "      <td>188</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>189</th>\n",
              "      <td>1</td>\n",
              "      <td>189</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>190</th>\n",
              "      <td>1</td>\n",
              "      <td>190</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>191</th>\n",
              "      <td>1</td>\n",
              "      <td>191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>192</th>\n",
              "      <td>1</td>\n",
              "      <td>192</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>193</th>\n",
              "      <td>1</td>\n",
              "      <td>193</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>194</th>\n",
              "      <td>1</td>\n",
              "      <td>194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>195</th>\n",
              "      <td>1</td>\n",
              "      <td>195</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>196</th>\n",
              "      <td>1</td>\n",
              "      <td>196</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>197</th>\n",
              "      <td>1</td>\n",
              "      <td>197</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>198</th>\n",
              "      <td>1</td>\n",
              "      <td>198</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>199</th>\n",
              "      <td>1</td>\n",
              "      <td>199</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>200</th>\n",
              "      <td>2</td>\n",
              "      <td>102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>201</th>\n",
              "      <td>2</td>\n",
              "      <td>200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202</th>\n",
              "      <td>2</td>\n",
              "      <td>201</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>203</th>\n",
              "      <td>2</td>\n",
              "      <td>202</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>204</th>\n",
              "      <td>2</td>\n",
              "      <td>92</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>205</th>\n",
              "      <td>2</td>\n",
              "      <td>203</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>206</th>\n",
              "      <td>2</td>\n",
              "      <td>204</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>207</th>\n",
              "      <td>2</td>\n",
              "      <td>205</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>208</th>\n",
              "      <td>2</td>\n",
              "      <td>113</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>209</th>\n",
              "      <td>2</td>\n",
              "      <td>206</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>210</th>\n",
              "      <td>2</td>\n",
              "      <td>207</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>211</th>\n",
              "      <td>2</td>\n",
              "      <td>208</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>212</th>\n",
              "      <td>2</td>\n",
              "      <td>209</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>2</td>\n",
              "      <td>210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>214</th>\n",
              "      <td>2</td>\n",
              "      <td>211</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>215</th>\n",
              "      <td>2</td>\n",
              "      <td>212</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>216</th>\n",
              "      <td>2</td>\n",
              "      <td>147</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>217</th>\n",
              "      <td>2</td>\n",
              "      <td>192</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>218</th>\n",
              "      <td>2</td>\n",
              "      <td>213</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>219</th>\n",
              "      <td>2</td>\n",
              "      <td>214</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>220</th>\n",
              "      <td>2</td>\n",
              "      <td>215</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     UserID  EmbedID\n",
              "180       1      180\n",
              "181       1      181\n",
              "182       1      182\n",
              "183       1      183\n",
              "184       1      184\n",
              "185       1      185\n",
              "186       1      186\n",
              "187       1      187\n",
              "188       1      188\n",
              "189       1      189\n",
              "190       1      190\n",
              "191       1      191\n",
              "192       1      192\n",
              "193       1      193\n",
              "194       1      194\n",
              "195       1      195\n",
              "196       1      196\n",
              "197       1      197\n",
              "198       1      198\n",
              "199       1      199\n",
              "200       2      102\n",
              "201       2      200\n",
              "202       2      201\n",
              "203       2      202\n",
              "204       2       92\n",
              "205       2      203\n",
              "206       2      204\n",
              "207       2      205\n",
              "208       2      113\n",
              "209       2      206\n",
              "210       2      207\n",
              "211       2      208\n",
              "212       2      209\n",
              "213       2      210\n",
              "214       2      211\n",
              "215       2      212\n",
              "216       2      147\n",
              "217       2      192\n",
              "218       2      213\n",
              "219       2      214\n",
              "220       2      215"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Df7jhFY8bbX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "my_data = openHdf.groupby('UserID')['EmbedID'].apply(list).values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DL5Q7L628uEe",
        "colab_type": "code",
        "outputId": "6a29d18a-78cb-4d43-c6dd-e42c6456f82d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "source": [
        "my_data"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([list([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199]),\n",
              "       list([102, 200, 201, 202, 92, 203, 204, 205, 113, 206, 207, 208, 209, 210, 211, 212, 147, 192, 213, 214, 215, 216, 217, 218, 219, 166, 220, 221, 222, 223, 224, 225, 226, 227, 188, 168, 228, 229, 230, 231, 232, 233, 70, 234, 235, 236, 237, 154]),\n",
              "       list([238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 42, 254, 255, 256, 257, 258, 259, 260]),\n",
              "       ..., list([27, 8331, 207, 2372]),\n",
              "       list([94, 223, 207, 109, 2958, 1574, 6090, 201, 84, 2437, 1211, 4944, 525, 2152, 10033, 710, 92, 635, 1187, 930, 35, 4437, 91, 2239, 1730, 3068, 27, 134, 1109, 2642, 989, 925, 2235, 3139, 646, 159, 11418, 218, 1290, 1095, 3005]),\n",
              "       list([1043, 2958, 2372, 164942, 2395, 67811, 94, 13963, 2399, 154, 8254, 2397, 9702, 506, 2068, 9731, 419669, 2437, 1190051, 92, 2478, 3453, 524766, 29261, 69349, 143, 2651, 920, 42706, 3468, 27, 8546, 559])],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FiV61IrdPg_f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del openHdf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NE2HW6llJZtC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_index = 0\n",
        "epoch_index = 0\n",
        "# recEpoch_indexA = 0 #Used to help keep store of the total number of epoches with the models\n",
        "\n",
        "d1 = LogUniform(a=1, b=numbBooks)\n",
        "\n",
        "def generate_batch(batch_size, posRate, negRate): \n",
        "  \n",
        "    global data_index, epoch_index\n",
        "    \n",
        "    batch = np.zeros(shape=(batch_size, posRate), dtype=np.int32) \n",
        "    labels = np.zeros(shape=(batch_size, 1), dtype=np.int32)\n",
        "    negs = np.zeros(shape=(batch_size, negRate), dtype=np.int32)\n",
        "\n",
        "    n=0\n",
        "    while n < batch_size:\n",
        "        firstpick = np.random.choice( len(my_data[data_index]), 1)[0]\n",
        "        labels[n] = my_data[data_index][firstpick]\n",
        "        batchSet = my_data[data_index][:firstpick]+my_data[data_index][firstpick+1:]\n",
        "        batch[n] = np.random.choice( batchSet , posRate)\n",
        "        excludes = np.concatenate( (labels[n], batch[n]), axis=None)\n",
        "\n",
        "        criterea0 = 0\n",
        "        while criterea0 < negRate: #just in case \n",
        "            aa = d1.rvs(negRate*3).astype(int) \n",
        "            # aa = d1.rvs(negRate*4).astype(int) -1\n",
        "            sampleCandidates = np.setdiff1d( aa, excludes )\n",
        "            criterea0 = sampleCandidates.shape[0]\n",
        "\n",
        "        negs[n] = np.random.choice( sampleCandidates, negRate, replace=False)\n",
        "\n",
        "        n = n+1\n",
        "        data_index = (data_index + 1) % len(my_data) #may have to do something like len my_data[:]\n",
        "        if data_index == 0:\n",
        "            epoch_index = epoch_index + 1\n",
        "            print('Completed %d Epochs' % epoch_index)\n",
        "    \n",
        "    return batch, labels, negs    \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0DsXP9kb8DE9",
        "colab_type": "code",
        "outputId": "de5e7d46-8f70-4b7c-c924-7e287d611f4d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "here, goes, negs = generate_batch(20, 2, 3) \n",
        "print('batch', here)\n",
        "print('labels', goes)\n",
        "print('negs', negs)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "batch [[107  57]\n",
            " [227 237]\n",
            " [238 250]\n",
            " [261 262]\n",
            " [311 358]\n",
            " [218 111]\n",
            " [473 473]\n",
            " [254 561]\n",
            " [587 568]\n",
            " [616 624]\n",
            " [648 657]\n",
            " [664 664]\n",
            " [714 216]\n",
            " [210 210]\n",
            " [746 758]\n",
            " [776 776]\n",
            " [887 844]\n",
            " [892 898]\n",
            " [930 930]\n",
            " [ 72 983]]\n",
            "labels [[139]\n",
            " [192]\n",
            " [260]\n",
            " [124]\n",
            " [313]\n",
            " [455]\n",
            " [472]\n",
            " [544]\n",
            " [577]\n",
            " [611]\n",
            " [638]\n",
            " [661]\n",
            " [694]\n",
            " [736]\n",
            " [768]\n",
            " [777]\n",
            " [863]\n",
            " [902]\n",
            " [938]\n",
            " [978]]\n",
            "negs [[   3734 2452227       4]\n",
            " [     54      36    1041]\n",
            " [    693   18915  448927]\n",
            " [  99931   56872      13]\n",
            " [  35808       1 1783325]\n",
            " [ 372779  237415   39032]\n",
            " [  45194     273   19165]\n",
            " [      9    1232  615156]\n",
            " [   2902    5316     286]\n",
            " [      9      27      37]\n",
            " [ 988960      74     313]\n",
            " [    375   12520  937319]\n",
            " [    245   80020      29]\n",
            " [      1      62       7]\n",
            " [ 326459     284  302935]\n",
            " [  93593 1465274       2]\n",
            " [     35     522      76]\n",
            " [ 604550 1619922    6501]\n",
            " [  30829  236680    1000]\n",
            " [   1995    1713    2529]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EN8lEIKJxmRF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class SkipGramModel(nn.Module):\n",
        "    \"\"\"Skip gram model of word2vec.\n",
        "    Attributes:\n",
        "        emb_size: Embedding size.\n",
        "        emb_dimention: Embedding dimention, typically from 50 to 500.\n",
        "        u_embedding: Embedding for center word.\n",
        "        v_embedding: Embedding for neibor words.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, batchSize, skipWindow):\n",
        "        \"\"\"Initialize model parameters.\n",
        "        Apply for two embedding layers.\n",
        "        Initialize layer weight\n",
        "        Args:\n",
        "            emb_size: Embedding size.\n",
        "            emb_dimention: Embedding dimention, typically from 50 to 500.\n",
        "        Returns:\n",
        "            None\n",
        "        \"\"\"\n",
        "        super(SkipGramModel, self).__init__()\n",
        "        self.emb_size = batchSize\n",
        "        self.skipWindow = skipWindow\n",
        "        self.emb_dimension = emb_dimension\n",
        "        self.batchSize = batchSize\n",
        "        self.u_embeddings = nn.Embedding(batchSize, emb_dimension, sparse=True).cuda()\n",
        "        self.v_embeddings = nn.Embedding( batchSize, emb_dimension, sparse=True).cuda()\n",
        "        # self.targets = torch.ones(34816).cud\n",
        "        self.init_emb()\n",
        "\n",
        "    def init_emb(self):\n",
        "        \"\"\"Initialize embedding weight like word2vec.\n",
        "        The u_embedding is a uniform distribution in [-0.5/em_size, 0.5/emb_size], and the elements of v_embedding are zeroes.\n",
        "        Returns:\n",
        "            None\n",
        "        \"\"\"\n",
        "        initrange = (2.0 / (numbBooks + self.emb_dimension)) ** 0.5 # Xavier init\n",
        "        self.u_embeddings.weight.data.uniform_(-initrange, initrange)\n",
        "        self.v_embeddings.weight.data.normal_(mean=0, std=math.sqrt(initrange))\n",
        "        self.lossFunction = nn.BCEWithLogitsLoss( reduction = 'none' )\n",
        "\n",
        "    def forward(self, pos_u, pos_v, neg_v, targets ):\n",
        "        \"\"\"Forward process.\n",
        "        As pytorch designed, all variables must be batch format, so all input of this method is a list of word id.\n",
        "        Args:\n",
        "            pos_u: list of center word ids for positive word pairs.\n",
        "            pos_v: list of neibor word ids for positive word pairs.\n",
        "            neg_u: list of center word ids for negative word pairs.\n",
        "            neg_v: list of neibor word ids for negative word pairs.\n",
        "        Returns:\n",
        "            Loss of this process, a pytorch variable.\n",
        "        \"\"\"\n",
        "        emb_u = self.u_embeddings(pos_u)\n",
        "        emb_v = self.v_embeddings(pos_v)\n",
        "        neg_emb_v = self.v_embeddings(neg_v)\n",
        "\n",
        "        scorePos = torch.bmm(emb_u, torch.transpose(emb_v, 1, 2)).squeeze()\n",
        "        scoreNeg = torch.bmm( emb_u , torch.transpose(neg_emb_v, 1, 2) ).squeeze()\n",
        "\n",
        "        #reduce to dot products for each set, and concatinate all the losses\n",
        "        #noticed that the sign change for the negative sample dot products\n",
        "        totalScores = torch.cat( ( scorePos.reshape(scorePos.shape[0], -1) , -scoreNeg.reshape(scoreNeg.shape[0], -1) ) , dim=1)\n",
        "\n",
        "        indLoss = self.lossFunction( totalScores, targets )\n",
        "        rowSum = torch.sum(indLoss, dim=1) #Sum all losses for each set\n",
        "        finalLoss = torch.mean(rowSum) #Average losses across batches \n",
        "\n",
        "        return finalLoss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jcf12Y-4x8OL",
        "colab_type": "code",
        "outputId": "07937eee-fd3b-4110-aab0-bb9cad897f97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "output_file_name = 'theOutpule.file'\n",
        "\n",
        "emb_dimension = 400\n",
        "batch_size = 128\n",
        "posRate = 4\n",
        "negRate = 128\n",
        "iterationsMax = 40000 #200001\n",
        "# initial_lr = 1.0\n",
        "\n",
        "skip_gram_model = SkipGramModel(batch_size*(2*posRate+negRate), emb_dimension)\n",
        "use_cuda = torch.cuda.is_available()\n",
        "print('use cuda ? ', use_cuda)\n",
        "if use_cuda:\n",
        "    skip_gram_model.cuda()\n",
        "\n",
        "optimizer = optim.Adagrad(\n",
        "    skip_gram_model.parameters())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "use cuda ?  True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7MGtvtYpPmou",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y97O--DDPoju",
        "colab_type": "code",
        "outputId": "02987bb5-f5c9-4d12-d66b-2e2f375871f9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "gc.collect()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "207"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wnhraRnEzpTM",
        "colab_type": "code",
        "outputId": "c33f982c-4dc7-4847-9e32-5d02592cbbcb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "!pip install SpeedTorch"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: SpeedTorch in /usr/local/lib/python3.6/dist-packages (0.0.3)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from SpeedTorch) (1.1.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from SpeedTorch) (1.16.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "knEYJP7TLn5A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import SpeedTorch"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qk_CnNcBACB2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "uEmbed_switcher = SpeedTorch.ModelFactory( skip_gram_model.u_embeddings, total_classes=numbBooks, embed_dimension=400 )\n",
        "vEmbed_switcher = SpeedTorch.ModelFactory( skip_gram_model.v_embeddings, total_classes=numbBooks, embed_dimension=400 )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AJI3MvoJABqq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "initrange = (2.0 / (numbBooks + emb_dimension)) ** 0.5 # Xavier init\n",
        "uEmbed_switcher.uniformDistributionInit( -initrange, initrange )\n",
        "vEmbed_switcher.normalDistributionInit( 0, math.sqrt(initrange) )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l1p5DYskLxAc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "uAdagrad_switcher = SpeedTorch.OptimizerFactory( optimizer, total_classes=numbBooks, embed_dimension=400, model=skip_gram_model, variable_name= 'u_embeddings', CPUPinn = True)\n",
        "vAdagrad_switcher = SpeedTorch.OptimizerFactory( optimizer, total_classes=numbBooks, embed_dimension=400, model=skip_gram_model, variable_name='v_embeddings' , CPUPinn = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MoTu-2zmLw-1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "uAdagrad_switcher.optInit()\n",
        "vAdagrad_switcher.optInit()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ziXthZwjL1Jk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 128\n",
        "labelsDummyInput = uEmbed_switcher.variableTransformer( batchSize=batch_size, posPerBatch=1)\n",
        "batchDummpyInput, negzDummyInput = vEmbed_switcher.variableTransformer( batchSize=batch_size, posPerBatch=4, negPerBatch=128 )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jn9Zvv4PxtXV",
        "colab_type": "code",
        "outputId": "59911e5c-0e8c-434d-9a23-85e5cf402f0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "runningLoss = 0\n",
        "runningStepTime = 0 \n",
        "batch_size = 128\n",
        "negSamp = 128\n",
        "window_size = 2\n",
        "\n",
        "targets = torch.ones( batch_size, window_size*2 + negSamp , dtype = torch.float32 ).cuda()\n",
        "\n",
        "batchTensor = torch.from_numpy(batchDummpyInput)\n",
        "LabelTensor = torch.from_numpy(labelsDummyInput)\n",
        "negTensor = torch.from_numpy(negzDummyInput)\n",
        "\n",
        "pos_u = Variable(torch.LongTensor(LabelTensor.long()))\n",
        "pos_v = Variable(torch.LongTensor(batchTensor.long()))\n",
        "neg_v = Variable(torch.LongTensor(negTensor.long()))\n",
        "\n",
        "if use_cuda:\n",
        "    pos_u = pos_u.cuda()\n",
        "    pos_v = pos_v.cuda()\n",
        "    neg_v = neg_v.cuda()\n",
        "\n",
        "for i in range(80000):\n",
        "    start = time.time()\n",
        "    batch, labels, negz = generate_batch(batch_size=batch_size, posRate=4, negRate= 128)\n",
        "\n",
        "    with torch.no_grad():\n",
        "        uEmbed_switcher.beforeForwardPass( labels )\n",
        "        vEmbed_switcher.beforeForwardPass( batch, negz )\n",
        "        uAdagrad_switcher.beforeForwardPass( labels )\n",
        "        vAdagrad_switcher.beforeForwardPass( batch, negz )\n",
        "\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    loss = skip_gram_model.forward(pos_u, pos_v, neg_v, targets)\n",
        "    runningLoss = runningLoss + loss.data.item()\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        uEmbed_switcher.afterOptimizerStep( labels )\n",
        "        vEmbed_switcher.afterOptimizerStep( batch, negz )\n",
        "        uAdagrad_switcher.afterOptimizerStep( labels )\n",
        "        vAdagrad_switcher.afterOptimizerStep( batch, negz )\n",
        "\n",
        "    end = time.time()\n",
        "    runningStepTime = runningStepTime + end - start\n",
        "\n",
        "    if i%100 == 0:\n",
        "        print('i is ', i)\n",
        "        print('loss is ', runningLoss/100)\n",
        "        print('Average step time is ', runningStepTime/100)\n",
        "        runningLoss = 0\n",
        "        runningStepTime = 0\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "i is  52000\n",
            "loss is  24.798086128234864\n",
            "Average step time is  0.07182470083236694\n",
            "i is  52100\n",
            "loss is  24.741167640686037\n",
            "Average step time is  0.07112337112426757\n",
            "i is  52200\n",
            "loss is  24.91937467575073\n",
            "Average step time is  0.0710398030281067\n",
            "i is  52300\n",
            "loss is  24.830597553253174\n",
            "Average step time is  0.07117151260375977\n",
            "i is  52400\n",
            "loss is  25.540252838134766\n",
            "Average step time is  0.07145098447799683\n",
            "i is  52500\n",
            "loss is  24.378208885192873\n",
            "Average step time is  0.07141171216964722\n",
            "i is  52600\n",
            "loss is  24.295829238891603\n",
            "Average step time is  0.07101068258285523\n",
            "i is  52700\n",
            "loss is  24.338541355133056\n",
            "Average step time is  0.07091912508010864\n",
            "i is  52800\n",
            "loss is  24.324116039276124\n",
            "Average step time is  0.07124611377716064\n",
            "i is  52900\n",
            "loss is  24.288192501068114\n",
            "Average step time is  0.07189893960952759\n",
            "i is  53000\n",
            "loss is  24.025996208190918\n",
            "Average step time is  0.07162424802780151\n",
            "i is  53100\n",
            "loss is  24.23704013824463\n",
            "Average step time is  0.07164643287658691\n",
            "i is  53200\n",
            "loss is  24.52386745452881\n",
            "Average step time is  0.07164447069168091\n",
            "i is  53300\n",
            "loss is  23.884730167388916\n",
            "Average step time is  0.07166831970214843\n",
            "i is  53400\n",
            "loss is  23.841606674194335\n",
            "Average step time is  0.0714738392829895\n",
            "i is  53500\n",
            "loss is  23.82346643447876\n",
            "Average step time is  0.07127052307128906\n",
            "i is  53600\n",
            "loss is  24.1445232963562\n",
            "Average step time is  0.0712718677520752\n",
            "i is  53700\n",
            "loss is  24.529640407562255\n",
            "Average step time is  0.07155957937240601\n",
            "i is  53800\n",
            "loss is  23.80582899093628\n",
            "Average step time is  0.07149592399597168\n",
            "i is  53900\n",
            "loss is  24.086092510223388\n",
            "Average step time is  0.0712360954284668\n",
            "i is  54000\n",
            "loss is  24.127616214752198\n",
            "Average step time is  0.07132700443267823\n",
            "i is  54100\n",
            "loss is  24.417256298065187\n",
            "Average step time is  0.07067386150360107\n",
            "i is  54200\n",
            "loss is  24.314077930450438\n",
            "Average step time is  0.07039454221725464\n",
            "i is  54300\n",
            "loss is  24.447426586151124\n",
            "Average step time is  0.07038193941116333\n",
            "i is  54400\n",
            "loss is  24.619402656555177\n",
            "Average step time is  0.07098019123077393\n",
            "i is  54500\n",
            "loss is  24.707425632476806\n",
            "Average step time is  0.07175151824951172\n",
            "i is  54600\n",
            "loss is  24.854297103881837\n",
            "Average step time is  0.07173649549484253\n",
            "i is  54700\n",
            "loss is  25.1135129737854\n",
            "Average step time is  0.07144433259963989\n",
            "Completed 10 Epochs\n",
            "i is  54800\n",
            "loss is  26.417523822784425\n",
            "Average step time is  0.07098331928253174\n",
            "i is  54900\n",
            "loss is  25.43920150756836\n",
            "Average step time is  0.07048170566558838\n",
            "i is  55000\n",
            "loss is  25.79023302078247\n",
            "Average step time is  0.07119455099105836\n",
            "i is  55100\n",
            "loss is  25.70554084777832\n",
            "Average step time is  0.07166833639144897\n",
            "i is  55200\n",
            "loss is  25.403196716308592\n",
            "Average step time is  0.07114987373352051\n",
            "i is  55300\n",
            "loss is  24.949287166595457\n",
            "Average step time is  0.07207521915435791\n",
            "i is  55400\n",
            "loss is  24.987305641174316\n",
            "Average step time is  0.0721753740310669\n",
            "i is  55500\n",
            "loss is  24.96130786895752\n",
            "Average step time is  0.07140320301055908\n",
            "i is  55600\n",
            "loss is  24.89737789154053\n",
            "Average step time is  0.07166666030883789\n",
            "i is  55700\n",
            "loss is  24.902601375579835\n",
            "Average step time is  0.07117259740829468\n",
            "i is  55800\n",
            "loss is  24.829621353149413\n",
            "Average step time is  0.07149104833602905\n",
            "i is  55900\n",
            "loss is  24.737400608062742\n",
            "Average step time is  0.07131035804748535\n",
            "i is  56000\n",
            "loss is  24.927890396118165\n",
            "Average step time is  0.08083511590957641\n",
            "i is  56100\n",
            "loss is  25.06744201660156\n",
            "Average step time is  0.07119472026824951\n",
            "i is  56200\n",
            "loss is  24.645966720581054\n",
            "Average step time is  0.07115442037582398\n",
            "i is  56300\n",
            "loss is  24.298472900390625\n",
            "Average step time is  0.0713190770149231\n",
            "i is  56400\n",
            "loss is  24.419996242523194\n",
            "Average step time is  0.07107281923294068\n",
            "i is  56500\n",
            "loss is  24.35365156173706\n",
            "Average step time is  0.07156316041946412\n",
            "i is  56600\n",
            "loss is  24.067489852905272\n",
            "Average step time is  0.07151888370513916\n",
            "i is  56700\n",
            "loss is  24.133011875152587\n",
            "Average step time is  0.07115054368972779\n",
            "i is  56800\n",
            "loss is  24.2345751953125\n",
            "Average step time is  0.07102627277374268\n",
            "i is  56900\n",
            "loss is  24.080156650543213\n",
            "Average step time is  0.07082589626312256\n",
            "i is  57000\n",
            "loss is  23.926615638732912\n",
            "Average step time is  0.07107248783111572\n",
            "i is  57100\n",
            "loss is  24.394292812347413\n",
            "Average step time is  0.07066622972488404\n",
            "i is  57200\n",
            "loss is  24.357655029296875\n",
            "Average step time is  0.07020108461380005\n",
            "i is  57300\n",
            "loss is  24.79630319595337\n",
            "Average step time is  0.07047646522521972\n",
            "i is  57400\n",
            "loss is  24.597101650238038\n",
            "Average step time is  0.07086649656295776\n",
            "i is  57500\n",
            "loss is  24.502896747589112\n",
            "Average step time is  0.0714800238609314\n",
            "i is  57600\n",
            "loss is  24.609438400268555\n",
            "Average step time is  0.07126412630081176\n",
            "i is  57700\n",
            "loss is  24.43414789199829\n",
            "Average step time is  0.07154485464096069\n",
            "i is  57800\n",
            "loss is  24.501969509124756\n",
            "Average step time is  0.07119569778442383\n",
            "i is  57900\n",
            "loss is  24.612184524536133\n",
            "Average step time is  0.07085048675537109\n",
            "i is  58000\n",
            "loss is  24.05717748641968\n",
            "Average step time is  0.07130216360092163\n",
            "i is  58100\n",
            "loss is  24.036288967132567\n",
            "Average step time is  0.07088091850280762\n",
            "i is  58200\n",
            "loss is  23.738108768463135\n",
            "Average step time is  0.0709696102142334\n",
            "i is  58300\n",
            "loss is  24.135478649139404\n",
            "Average step time is  0.07095856428146362\n",
            "i is  58400\n",
            "loss is  23.895766372680665\n",
            "Average step time is  0.07058971881866455\n",
            "i is  58500\n",
            "loss is  23.90232173919678\n",
            "Average step time is  0.07109283685684203\n",
            "i is  58600\n",
            "loss is  24.179599075317384\n",
            "Average step time is  0.07088064908981323\n",
            "i is  58700\n",
            "loss is  23.611278820037843\n",
            "Average step time is  0.07142760038375855\n",
            "i is  58800\n",
            "loss is  23.953567485809327\n",
            "Average step time is  0.0713239574432373\n",
            "i is  58900\n",
            "loss is  23.60655101776123\n",
            "Average step time is  0.07140918970108032\n",
            "i is  59000\n",
            "loss is  24.002394409179686\n",
            "Average step time is  0.0704615569114685\n",
            "i is  59100\n",
            "loss is  23.883099193573\n",
            "Average step time is  0.07015539407730102\n",
            "i is  59200\n",
            "loss is  23.97181968688965\n",
            "Average step time is  0.07112930297851562\n",
            "i is  59300\n",
            "loss is  23.468880138397218\n",
            "Average step time is  0.07156620740890503\n",
            "i is  59400\n",
            "loss is  23.505080738067626\n",
            "Average step time is  0.0707843017578125\n",
            "i is  59500\n",
            "loss is  23.963476409912108\n",
            "Average step time is  0.07025428533554078\n",
            "i is  59600\n",
            "loss is  23.99349397659302\n",
            "Average step time is  0.07084404706954955\n",
            "i is  59700\n",
            "loss is  23.820845756530762\n",
            "Average step time is  0.07193159580230712\n",
            "i is  59800\n",
            "loss is  24.211629390716553\n",
            "Average step time is  0.0714738941192627\n",
            "i is  59900\n",
            "loss is  24.10433448791504\n",
            "Average step time is  0.07194942951202393\n",
            "i is  60000\n",
            "loss is  24.414105911254882\n",
            "Average step time is  0.07136979103088378\n",
            "i is  60100\n",
            "loss is  24.498632678985597\n",
            "Average step time is  0.0717807388305664\n",
            "i is  60200\n",
            "loss is  24.859443454742433\n",
            "Average step time is  0.07116961240768432\n",
            "Completed 11 Epochs\n",
            "i is  60300\n",
            "loss is  26.064680309295653\n",
            "Average step time is  0.07060210466384888\n",
            "i is  60400\n",
            "loss is  25.140376014709474\n",
            "Average step time is  0.0713640022277832\n",
            "i is  60500\n",
            "loss is  25.38657123565674\n",
            "Average step time is  0.07062175035476685\n",
            "i is  60600\n",
            "loss is  25.262399616241456\n",
            "Average step time is  0.07038785696029663\n",
            "i is  60700\n",
            "loss is  24.80905258178711\n",
            "Average step time is  0.07102229118347168\n",
            "i is  60800\n",
            "loss is  24.632071762084962\n",
            "Average step time is  0.07096128225326538\n",
            "i is  60900\n",
            "loss is  24.768754024505615\n",
            "Average step time is  0.07069090127944946\n",
            "i is  61000\n",
            "loss is  24.568951988220213\n",
            "Average step time is  0.07033885002136231\n",
            "i is  61100\n",
            "loss is  24.258803539276123\n",
            "Average step time is  0.07993866920471192\n",
            "i is  61200\n",
            "loss is  24.22540271759033\n",
            "Average step time is  0.07054737806320191\n",
            "i is  61300\n",
            "loss is  24.772471408843995\n",
            "Average step time is  0.07034996271133423\n",
            "i is  61400\n",
            "loss is  23.964405250549316\n",
            "Average step time is  0.0701241397857666\n",
            "i is  61500\n",
            "loss is  24.589141750335692\n",
            "Average step time is  0.07043203115463256\n",
            "i is  61600\n",
            "loss is  24.375861644744873\n",
            "Average step time is  0.07041082859039306\n",
            "i is  61700\n",
            "loss is  24.399599456787108\n",
            "Average step time is  0.07070482969284057\n",
            "i is  61800\n",
            "loss is  24.291701469421387\n",
            "Average step time is  0.07053585290908813\n",
            "i is  61900\n",
            "loss is  24.3153311920166\n",
            "Average step time is  0.07056767225265503\n",
            "i is  62000\n",
            "loss is  24.100422306060793\n",
            "Average step time is  0.07069825649261474\n",
            "i is  62100\n",
            "loss is  23.701162872314452\n",
            "Average step time is  0.07071583986282348\n",
            "i is  62200\n",
            "loss is  23.967095184326173\n",
            "Average step time is  0.07066250562667847\n",
            "i is  62300\n",
            "loss is  23.92034418106079\n",
            "Average step time is  0.07032194375991821\n",
            "i is  62400\n",
            "loss is  23.750561981201173\n",
            "Average step time is  0.07038239002227784\n",
            "i is  62500\n",
            "loss is  23.881969661712645\n",
            "Average step time is  0.07022981882095337\n",
            "i is  62600\n",
            "loss is  23.941547241210937\n",
            "Average step time is  0.0704434084892273\n",
            "i is  62700\n",
            "loss is  24.352622947692872\n",
            "Average step time is  0.07047839879989624\n",
            "i is  62800\n",
            "loss is  24.420960311889647\n",
            "Average step time is  0.07057724475860595\n",
            "i is  62900\n",
            "loss is  24.29303985595703\n",
            "Average step time is  0.0705711555480957\n",
            "i is  63000\n",
            "loss is  24.081491355895995\n",
            "Average step time is  0.07110551357269287\n",
            "i is  63100\n",
            "loss is  24.100899753570555\n",
            "Average step time is  0.07031147718429566\n",
            "i is  63200\n",
            "loss is  24.326708545684813\n",
            "Average step time is  0.07048182487487793\n",
            "i is  63300\n",
            "loss is  24.371923122406006\n",
            "Average step time is  0.07003509521484375\n",
            "i is  63400\n",
            "loss is  24.249788970947264\n",
            "Average step time is  0.0706062936782837\n",
            "i is  63500\n",
            "loss is  23.74524923324585\n",
            "Average step time is  0.07039709568023682\n",
            "i is  63600\n",
            "loss is  23.76484571456909\n",
            "Average step time is  0.07082278966903686\n",
            "i is  63700\n",
            "loss is  23.438884487152098\n",
            "Average step time is  0.07030854701995849\n",
            "i is  63800\n",
            "loss is  23.689725952148436\n",
            "Average step time is  0.07055381059646607\n",
            "i is  63900\n",
            "loss is  23.839447631835938\n",
            "Average step time is  0.07060682773590088\n",
            "i is  64000\n",
            "loss is  23.656782665252685\n",
            "Average step time is  0.07084453105926514\n",
            "i is  64100\n",
            "loss is  23.755608081817627\n",
            "Average step time is  0.07116249084472656\n",
            "i is  64200\n",
            "loss is  23.525827980041505\n",
            "Average step time is  0.07067011594772339\n",
            "i is  64300\n",
            "loss is  23.58279914855957\n",
            "Average step time is  0.07171991586685181\n",
            "i is  64400\n",
            "loss is  23.52138864517212\n",
            "Average step time is  0.0716361165046692\n",
            "i is  64500\n",
            "loss is  23.281515712738038\n",
            "Average step time is  0.07064810514450073\n",
            "i is  64600\n",
            "loss is  23.67943286895752\n",
            "Average step time is  0.07083216428756714\n",
            "i is  64700\n",
            "loss is  23.61856061935425\n",
            "Average step time is  0.07172291755676269\n",
            "i is  64800\n",
            "loss is  23.496794013977052\n",
            "Average step time is  0.07106835365295411\n",
            "i is  64900\n",
            "loss is  23.43643041610718\n",
            "Average step time is  0.07101618051528931\n",
            "i is  65000\n",
            "loss is  23.784302158355715\n",
            "Average step time is  0.07044194221496582\n",
            "i is  65100\n",
            "loss is  23.784012928009034\n",
            "Average step time is  0.07098864555358887\n",
            "i is  65200\n",
            "loss is  23.487497844696044\n",
            "Average step time is  0.07128032445907592\n",
            "i is  65300\n",
            "loss is  23.526673622131348\n",
            "Average step time is  0.07121106624603271\n",
            "i is  65400\n",
            "loss is  23.85608268737793\n",
            "Average step time is  0.07081825017929078\n",
            "i is  65500\n",
            "loss is  24.029944763183593\n",
            "Average step time is  0.0705692458152771\n",
            "i is  65600\n",
            "loss is  24.299066104888915\n",
            "Average step time is  0.0710855221748352\n",
            "Completed 12 Epochs\n",
            "i is  65700\n",
            "loss is  25.24775547027588\n",
            "Average step time is  0.07107762813568115\n",
            "i is  65800\n",
            "loss is  24.904204959869386\n",
            "Average step time is  0.0708454704284668\n",
            "i is  65900\n",
            "loss is  24.656728267669678\n",
            "Average step time is  0.07210709571838379\n",
            "i is  66000\n",
            "loss is  24.83049018859863\n",
            "Average step time is  0.07163892269134521\n",
            "i is  66100\n",
            "loss is  25.083869400024415\n",
            "Average step time is  0.07213754415512084\n",
            "i is  66200\n",
            "loss is  24.27948797225952\n",
            "Average step time is  0.0809386444091797\n",
            "i is  66300\n",
            "loss is  24.25234306335449\n",
            "Average step time is  0.07114956617355346\n",
            "i is  66400\n",
            "loss is  24.543215847015382\n",
            "Average step time is  0.07129941225051879\n",
            "i is  66500\n",
            "loss is  24.227126750946045\n",
            "Average step time is  0.07064841032028198\n",
            "i is  66600\n",
            "loss is  24.22827865600586\n",
            "Average step time is  0.07105532884597779\n",
            "i is  66700\n",
            "loss is  24.031573848724364\n",
            "Average step time is  0.07141144275665283\n",
            "i is  66800\n",
            "loss is  24.27535966873169\n",
            "Average step time is  0.07161723613739014\n",
            "i is  66900\n",
            "loss is  23.95821973800659\n",
            "Average step time is  0.07165347099304199\n",
            "i is  67000\n",
            "loss is  24.419406566619873\n",
            "Average step time is  0.07140408277511597\n",
            "i is  67100\n",
            "loss is  24.12562791824341\n",
            "Average step time is  0.0705480432510376\n",
            "i is  67200\n",
            "loss is  24.024035263061524\n",
            "Average step time is  0.07114908695220948\n",
            "i is  67300\n",
            "loss is  23.992862148284914\n",
            "Average step time is  0.07135263919830322\n",
            "i is  67400\n",
            "loss is  23.820624256134032\n",
            "Average step time is  0.07175800561904908\n",
            "i is  67500\n",
            "loss is  23.853535957336426\n",
            "Average step time is  0.07114603996276855\n",
            "i is  67600\n",
            "loss is  23.49094358444214\n",
            "Average step time is  0.07073957920074463\n",
            "i is  67700\n",
            "loss is  23.53730100631714\n",
            "Average step time is  0.07109804153442383\n",
            "i is  67800\n",
            "loss is  23.70723352432251\n",
            "Average step time is  0.07070603847503662\n",
            "i is  67900\n",
            "loss is  23.19246603012085\n",
            "Average step time is  0.07071868658065796\n",
            "i is  68000\n",
            "loss is  23.73000701904297\n",
            "Average step time is  0.07176758050918579\n",
            "i is  68100\n",
            "loss is  23.85187473297119\n",
            "Average step time is  0.0708172607421875\n",
            "i is  68200\n",
            "loss is  23.998873443603514\n",
            "Average step time is  0.0716014289855957\n",
            "i is  68300\n",
            "loss is  24.063264427185057\n",
            "Average step time is  0.0717323350906372\n",
            "i is  68400\n",
            "loss is  23.824221286773682\n",
            "Average step time is  0.07114521265029908\n",
            "i is  68500\n",
            "loss is  23.765509510040282\n",
            "Average step time is  0.07108657598495484\n",
            "i is  68600\n",
            "loss is  23.7397700881958\n",
            "Average step time is  0.07127634048461914\n",
            "i is  68700\n",
            "loss is  23.596688804626464\n",
            "Average step time is  0.07144954919815064\n",
            "i is  68800\n",
            "loss is  23.97802080154419\n",
            "Average step time is  0.07171725034713745\n",
            "i is  68900\n",
            "loss is  23.850891361236574\n",
            "Average step time is  0.07129854679107667\n",
            "i is  69000\n",
            "loss is  23.21173542022705\n",
            "Average step time is  0.07038565158843994\n",
            "i is  69100\n",
            "loss is  23.1682967376709\n",
            "Average step time is  0.07056692361831665\n",
            "i is  69200\n",
            "loss is  23.379605102539063\n",
            "Average step time is  0.07053793668746948\n",
            "i is  69300\n",
            "loss is  23.60899293899536\n",
            "Average step time is  0.07058803558349609\n",
            "i is  69400\n",
            "loss is  23.474609203338623\n",
            "Average step time is  0.07043023824691773\n",
            "i is  69500\n",
            "loss is  23.256080837249755\n",
            "Average step time is  0.0708016300201416\n",
            "i is  69600\n",
            "loss is  23.344483299255373\n",
            "Average step time is  0.07078628778457642\n",
            "i is  69700\n",
            "loss is  23.20562246322632\n",
            "Average step time is  0.07067926406860352\n",
            "i is  69800\n",
            "loss is  23.529333724975587\n",
            "Average step time is  0.07064794778823852\n",
            "i is  69900\n",
            "loss is  23.062298336029052\n",
            "Average step time is  0.07074851512908936\n",
            "i is  70000\n",
            "loss is  23.35726234436035\n",
            "Average step time is  0.07029223442077637\n",
            "i is  70100\n",
            "loss is  23.58796646118164\n",
            "Average step time is  0.07035402536392212\n",
            "i is  70200\n",
            "loss is  23.320481548309328\n",
            "Average step time is  0.07024515390396119\n",
            "i is  70300\n",
            "loss is  23.04322166442871\n",
            "Average step time is  0.07097436189651489\n",
            "i is  70400\n",
            "loss is  23.09322645187378\n",
            "Average step time is  0.07091015577316284\n",
            "i is  70500\n",
            "loss is  23.712051525115967\n",
            "Average step time is  0.07062340259552002\n",
            "i is  70600\n",
            "loss is  23.22908981323242\n",
            "Average step time is  0.07050175666809082\n",
            "i is  70700\n",
            "loss is  23.560459423065186\n",
            "Average step time is  0.0701702332496643\n",
            "i is  70800\n",
            "loss is  23.26126724243164\n",
            "Average step time is  0.07123926401138306\n",
            "i is  70900\n",
            "loss is  23.650171947479247\n",
            "Average step time is  0.07131480455398559\n",
            "i is  71000\n",
            "loss is  23.9893013381958\n",
            "Average step time is  0.07088452816009522\n",
            "i is  71100\n",
            "loss is  24.001534271240235\n",
            "Average step time is  0.07095285654067993\n",
            "Completed 13 Epochs\n",
            "i is  71200\n",
            "loss is  25.00287805557251\n",
            "Average step time is  0.07094946384429932\n",
            "i is  71300\n",
            "loss is  24.407639465332032\n",
            "Average step time is  0.08075182914733886\n",
            "i is  71400\n",
            "loss is  24.220629978179932\n",
            "Average step time is  0.07120179414749145\n",
            "i is  71500\n",
            "loss is  24.77134859085083\n",
            "Average step time is  0.07151468753814698\n",
            "i is  71600\n",
            "loss is  24.50631782531738\n",
            "Average step time is  0.07153103828430175\n",
            "i is  71700\n",
            "loss is  24.143973579406737\n",
            "Average step time is  0.07194599628448486\n",
            "i is  71800\n",
            "loss is  23.821280307769776\n",
            "Average step time is  0.07187033414840699\n",
            "i is  71900\n",
            "loss is  23.900002517700194\n",
            "Average step time is  0.0713994026184082\n",
            "i is  72000\n",
            "loss is  23.835079345703125\n",
            "Average step time is  0.07099872827529907\n",
            "i is  72100\n",
            "loss is  23.97481050491333\n",
            "Average step time is  0.07118072748184204\n",
            "i is  72200\n",
            "loss is  23.688680992126464\n",
            "Average step time is  0.07157530307769776\n",
            "i is  72300\n",
            "loss is  23.70778917312622\n",
            "Average step time is  0.07168420314788819\n",
            "i is  72400\n",
            "loss is  23.92837772369385\n",
            "Average step time is  0.07199958324432373\n",
            "i is  72500\n",
            "loss is  23.89516658782959\n",
            "Average step time is  0.0714749550819397\n",
            "i is  72600\n",
            "loss is  23.680802955627442\n",
            "Average step time is  0.07075593948364257\n",
            "i is  72700\n",
            "loss is  23.695276107788086\n",
            "Average step time is  0.07134311199188233\n",
            "i is  72800\n",
            "loss is  23.53389757156372\n",
            "Average step time is  0.0715197515487671\n",
            "i is  72900\n",
            "loss is  23.428748512268065\n",
            "Average step time is  0.07172432661056519\n",
            "i is  73000\n",
            "loss is  23.297233390808106\n",
            "Average step time is  0.07108497858047486\n",
            "i is  73100\n",
            "loss is  23.286897258758543\n",
            "Average step time is  0.07135836124420165\n",
            "i is  73200\n",
            "loss is  23.234059238433836\n",
            "Average step time is  0.07129199981689453\n",
            "i is  73300\n",
            "loss is  23.226083126068115\n",
            "Average step time is  0.07085265874862672\n",
            "i is  73400\n",
            "loss is  23.06865167617798\n",
            "Average step time is  0.07154086112976074\n",
            "i is  73500\n",
            "loss is  23.365487232208253\n",
            "Average step time is  0.07127605676651001\n",
            "i is  73600\n",
            "loss is  23.753154754638672\n",
            "Average step time is  0.07155242443084717\n",
            "i is  73700\n",
            "loss is  23.686430778503418\n",
            "Average step time is  0.07129487752914429\n",
            "i is  73800\n",
            "loss is  24.032839393615724\n",
            "Average step time is  0.07179658651351929\n",
            "i is  73900\n",
            "loss is  23.46457759857178\n",
            "Average step time is  0.07159231662750244\n",
            "i is  74000\n",
            "loss is  23.542836799621583\n",
            "Average step time is  0.07136735916137696\n",
            "i is  74100\n",
            "loss is  23.795552768707275\n",
            "Average step time is  0.07144821166992188\n",
            "i is  74200\n",
            "loss is  23.74592821121216\n",
            "Average step time is  0.07148978710174561\n",
            "i is  74300\n",
            "loss is  23.64311435699463\n",
            "Average step time is  0.07209330558776855\n",
            "i is  74400\n",
            "loss is  23.267421283721923\n",
            "Average step time is  0.07131664276123047\n",
            "i is  74500\n",
            "loss is  23.231512832641602\n",
            "Average step time is  0.07142509698867798\n",
            "i is  74600\n",
            "loss is  22.96632703781128\n",
            "Average step time is  0.07146267890930176\n",
            "i is  74700\n",
            "loss is  23.18826894760132\n",
            "Average step time is  0.07169141530990601\n",
            "i is  74800\n",
            "loss is  23.477759742736815\n",
            "Average step time is  0.07160043954849243\n",
            "i is  74900\n",
            "loss is  23.072783946990967\n",
            "Average step time is  0.07209215641021728\n",
            "i is  75000\n",
            "loss is  23.194696826934816\n",
            "Average step time is  0.07155982971191406\n",
            "i is  75100\n",
            "loss is  23.059715309143066\n",
            "Average step time is  0.07211647033691407\n",
            "i is  75200\n",
            "loss is  22.833625354766845\n",
            "Average step time is  0.07135648965835571\n",
            "i is  75300\n",
            "loss is  22.987621574401857\n",
            "Average step time is  0.0713672661781311\n",
            "i is  75400\n",
            "loss is  22.855640125274657\n",
            "Average step time is  0.07176472663879395\n",
            "i is  75500\n",
            "loss is  22.917644844055175\n",
            "Average step time is  0.07129724264144897\n",
            "i is  75600\n",
            "loss is  23.241768760681154\n",
            "Average step time is  0.07181361436843872\n",
            "i is  75700\n",
            "loss is  22.859730377197266\n",
            "Average step time is  0.0716169810295105\n",
            "i is  75800\n",
            "loss is  22.96032859802246\n",
            "Average step time is  0.07169314622879028\n",
            "i is  75900\n",
            "loss is  23.469977016448976\n",
            "Average step time is  0.07152934074401855\n",
            "i is  76000\n",
            "loss is  23.392993602752686\n",
            "Average step time is  0.07152206897735595\n",
            "i is  76100\n",
            "loss is  23.25173442840576\n",
            "Average step time is  0.07182425260543823\n",
            "i is  76200\n",
            "loss is  23.354477138519286\n",
            "Average step time is  0.07140336275100707\n",
            "i is  76300\n",
            "loss is  23.310545120239258\n",
            "Average step time is  0.07168193578720093\n",
            "i is  76400\n",
            "loss is  23.16520450592041\n",
            "Average step time is  0.08094491004943848\n",
            "i is  76500\n",
            "loss is  23.63122241973877\n",
            "Average step time is  0.07183035612106323\n",
            "i is  76600\n",
            "loss is  24.033037147521974\n",
            "Average step time is  0.07176181316375732\n",
            "Completed 14 Epochs\n",
            "i is  76700\n",
            "loss is  25.087419090270995\n",
            "Average step time is  0.07166502952575683\n",
            "i is  76800\n",
            "loss is  24.30775789260864\n",
            "Average step time is  0.07156320571899415\n",
            "i is  76900\n",
            "loss is  24.4077988243103\n",
            "Average step time is  0.07195961475372314\n",
            "i is  77000\n",
            "loss is  24.537756423950196\n",
            "Average step time is  0.07193487405776977\n",
            "i is  77100\n",
            "loss is  24.12040719985962\n",
            "Average step time is  0.07163776636123657\n",
            "i is  77200\n",
            "loss is  23.74938835144043\n",
            "Average step time is  0.07171162128448487\n",
            "i is  77300\n",
            "loss is  23.674030590057374\n",
            "Average step time is  0.07116926193237305\n",
            "i is  77400\n",
            "loss is  23.665047817230224\n",
            "Average step time is  0.07197036027908325\n",
            "i is  77500\n",
            "loss is  23.293283939361572\n",
            "Average step time is  0.07163304328918457\n",
            "i is  77600\n",
            "loss is  23.88493757247925\n",
            "Average step time is  0.07155833005905152\n",
            "i is  77700\n",
            "loss is  23.581946392059326\n",
            "Average step time is  0.07133625507354736\n",
            "i is  77800\n",
            "loss is  23.45753107070923\n",
            "Average step time is  0.07161438226699829\n",
            "i is  77900\n",
            "loss is  23.66832134246826\n",
            "Average step time is  0.072230703830719\n",
            "i is  78000\n",
            "loss is  23.55891944885254\n",
            "Average step time is  0.0718804383277893\n",
            "i is  78100\n",
            "loss is  23.63354679107666\n",
            "Average step time is  0.07213417053222657\n",
            "i is  78200\n",
            "loss is  23.196320285797118\n",
            "Average step time is  0.07159191131591797\n",
            "i is  78300\n",
            "loss is  23.31555358886719\n",
            "Average step time is  0.0716717529296875\n",
            "i is  78400\n",
            "loss is  23.26194326400757\n",
            "Average step time is  0.07176347494125367\n",
            "i is  78500\n",
            "loss is  23.044481410980225\n",
            "Average step time is  0.07144588947296143\n",
            "i is  78600\n",
            "loss is  22.95625415802002\n",
            "Average step time is  0.07137244939804077\n",
            "i is  78700\n",
            "loss is  23.0755499458313\n",
            "Average step time is  0.07161298274993896\n",
            "i is  78800\n",
            "loss is  22.949783573150636\n",
            "Average step time is  0.07145041227340698\n",
            "i is  78900\n",
            "loss is  22.798595314025878\n",
            "Average step time is  0.07181024789810181\n",
            "i is  79000\n",
            "loss is  23.158511905670167\n",
            "Average step time is  0.07160574913024903\n",
            "i is  79100\n",
            "loss is  23.24155014038086\n",
            "Average step time is  0.07176677227020263\n",
            "i is  79200\n",
            "loss is  23.595177898406984\n",
            "Average step time is  0.07201067924499512\n",
            "i is  79300\n",
            "loss is  23.500386543273926\n",
            "Average step time is  0.07229062795639038\n",
            "i is  79400\n",
            "loss is  23.525770092010497\n",
            "Average step time is  0.07219119787216187\n",
            "i is  79500\n",
            "loss is  23.395408706665037\n",
            "Average step time is  0.07254691123962402\n",
            "i is  79600\n",
            "loss is  23.675126819610597\n",
            "Average step time is  0.0722329306602478\n",
            "i is  79700\n",
            "loss is  23.25771827697754\n",
            "Average step time is  0.07189785480499268\n",
            "i is  79800\n",
            "loss is  23.381868362426758\n",
            "Average step time is  0.07188498258590698\n",
            "i is  79900\n",
            "loss is  23.05620449066162\n",
            "Average step time is  0.07178986549377442\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XDjt0nXLz071",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# i is  98500\n",
        "# loss is  23.02226722717285\n",
        "# Average step time is  0.07153946399688721\n",
        "# Completed 18 Epochs\n",
        "# i is  98600"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yVeXYszF5C7U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del uAdagrad_switcher\n",
        "del vAdagrad_switcher"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YlTualwp75od",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del my_data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GyeIR7hQa-bK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del vEmbed_switcher"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lvXtMOGt7szG",
        "colab_type": "code",
        "outputId": "74155cad-8ad7-4c6c-a7f1-11b346606602",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "gc.collect()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "395"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lz8KCSnMIeAh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "numpyU = uEmbed_switcher.getNumpyVersion()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R6AS9Ujbae9t",
        "colab_type": "code",
        "outputId": "29bfd9f5-9cd1-4f32-edcf-e964d90449e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "numpyU"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-1.2862083e-01,  8.5094482e-02, -3.0265175e-02, ...,\n",
              "        -8.2676597e-03, -2.9408952e-02,  1.8072788e-01],\n",
              "       [ 2.3332495e-02,  3.0581463e-02, -3.4003556e-02, ...,\n",
              "         2.9853960e-02,  3.2352291e-02, -1.8195402e-02],\n",
              "       [ 1.2882236e-02,  1.4202858e-02, -1.3527294e-02, ...,\n",
              "         1.4110753e-02,  1.3384230e-02, -1.0938700e-02],\n",
              "       ...,\n",
              "       [ 1.7646885e-04,  6.0551678e-04, -7.8915490e-04, ...,\n",
              "         2.9078254e-04, -2.0462749e-04,  4.5109168e-04],\n",
              "       [ 1.0356624e-02,  9.7975582e-03, -1.0615848e-02, ...,\n",
              "         1.0558417e-02,  9.5852371e-03, -9.4440542e-03],\n",
              "       [ 3.2403204e-05,  2.6075924e-05, -2.3735006e-04, ...,\n",
              "        -1.0021922e-04,  7.6251494e-04,  1.8066516e-05]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IkXMlEfJ5uvB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "numpyU.shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xlk0cms6RxV2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# uEmbed_switcher.saveCupy( 'uEmbedsFinal.npy.cpy')\n",
        "# # vEmbed_switcher.saveCupy( 'vEmbedsFinal.npy.cpy')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uR-AuhEHaksk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# np.save('testFile.npy',numpyU )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T0w6wIcGx1nB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UHizOWr4x3_d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "np.save('Lit2vec2p8Mil.npy', numpyU )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-TBaNtHdqmR0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# num_points = 400\n",
        "\n",
        "# tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')\n",
        "# two_d_embeddings = tsne.fit_transform(final_embeddings[1:num_points+1, :])\n",
        "# two_d_embeddingsSM = tsne.fit_transform(final_embeddingsSM[1:num_points+1, :])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Un02WDWUBr1I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# def plot(embeddings, labels):\n",
        "#   assert embeddings.shape[0] >= len(labels), 'More labels than embeddings'\n",
        "#   pylab.figure(figsize=(50,50))  # in inches\n",
        "#   for i, label in enumerate(labels):\n",
        "#     x, y = embeddings[i,:]\n",
        "#     pylab.scatter(x, y)\n",
        "#     pylab.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
        "#                    ha='right', va='bottom')\n",
        "#   pylab.show()\n",
        "\n",
        "# books = [bookDictionary[i] for i in range(1, num_points+1)]\n",
        "# plot(two_d_embeddings, books)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IKd4EmLZEtD2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# def plot(softmax_weightsPlot, labels):\n",
        "#   assert softmax_weightsPlot.shape[0] >= len(labels), 'More labels than embeddings'\n",
        "#   pylab.figure(figsize=(50,50))  # in inches\n",
        "#   for i, label in enumerate(labels):\n",
        "#     x, y = softmax_weightsPlot[i,:]\n",
        "#     pylab.scatter(x, y)\n",
        "#     pylab.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
        "#                    ha='right', va='bottom')\n",
        "#   pylab.show()\n",
        "\n",
        "# books = [bookDictionary[i] for i in range(1, num_points+1)]\n",
        "# plot(two_d_embeddingsSM, books)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}