arxiv_paper_download_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# #Readme\n",
    "\n",
    "This ipython notebook allows to download all [arxiv metadata](https://arxiv.org/help/oa/index) using the [OAI standard](http://www.openarchives.org/OAI/2.0/openarchivesprotocol.htm) and _pyoai_ python package:  \n",
    "`pip install pyoai`. The pyoai api is described on [git](https://github.com/infrae/pyoai/blob/master/doc/API.html)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## import packages\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "# conda install -c anaconda beautiful-soup=4.3.2\n",
    "# conda install lxml\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import datetime\n",
    "import dateutil\n",
    "\n",
    "from oaipmh.client import Client\n",
    "from oaipmh.metadata import MetadataRegistry, oai_dc_reader\n",
    "# pip install pyoai\n",
    "\n",
    "import json\n",
    "import os\n",
    "\n",
    "import requests\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### set os path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let's checkout first metadata file\n",
    "# default_path = r'C:\\Users\\user\\Downloads\\arxvivdata'\n",
    "default_path = r'C:\\Users\\user\\Downloads\\arxivdata' \n",
    "os.chdir(default_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### initalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# service base url\n",
    "url = \"http://export.arxiv.org/oai2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## set up server\n",
    "registry = MetadataRegistry()\n",
    "registry.registerReader('oai_dc', oai_dc_reader)\n",
    "client = Client(url, registry)\n",
    "client.updateGranularity()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arXiv\n",
      "http://export.arxiv.org/oai2\n",
      "2.0\n"
     ]
    }
   ],
   "source": [
    "## get server info \n",
    "serverinfo = client.identify()\n",
    "print(serverinfo.repositoryName())\n",
    "print(serverinfo.baseURL())\n",
    "print(serverinfo.protocolVersion())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for info see https://arxiv.org/help/oa/index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('oai_dc',\n",
       "  'http://www.openarchives.org/OAI/2.0/oai_dc.xsd',\n",
       "  'http://www.openarchives.org/OAI/2.0/oai_dc/'),\n",
       " ('arXiv', 'http://arxiv.org/OAI/arXiv.xsd', 'http://arxiv.org/OAI/arXiv/'),\n",
       " ('arXivOld',\n",
       "  'http://arxiv.org/OAI/arXivOld.xsd',\n",
       "  'http://arxiv.org/OAI/arXivOld/'),\n",
       " ('arXivRaw',\n",
       "  'http://arxiv.org/OAI/arXivRaw.xsd',\n",
       "  'http://arxiv.org/OAI/arXivRaw/')]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check metadata formats\n",
    "client.listMetadataFormats()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get single record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get single record\n",
    "id = 'oai:arXiv.org:1610.02097'\n",
    "# need to use oai_dc as metadata format\n",
    "metapre = 'oai_dc'\n",
    "record = client.getRecord(identifier = id, metadataPrefix=metapre)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oai:arXiv.org:1610.02097\n",
      "2017-05-16 00:00:00\n"
     ]
    }
   ],
   "source": [
    "# header is element 0 and contains info about record\n",
    "header = record[0]\n",
    "print(header.identifier())\n",
    "print(header.datestamp())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contributor': [],\n",
       " 'coverage': [],\n",
       " 'creator': ['Jaskula, Jean-Christophe',\n",
       "  'Bauch, Erik',\n",
       "  'Arroyo-Camejo, Silvia',\n",
       "  'Lukin, Mikhail D.',\n",
       "  'Hell, Stefan W.',\n",
       "  'Trifonov, Alexei S.',\n",
       "  'Walsworth, Ronald L.'],\n",
       " 'date': ['2016-10-06'],\n",
       " 'description': ['  Nitrogen vacancy (NV) color centers in diamond are a leading modality for\\nboth superresolution optical imaging and nanoscale magnetic field sensing. In\\nthis work, we solve the remaining key challenge of performing optical magnetic\\nimaging and spectroscopy selectively on multiple NV centers that are located\\nwithin a diffraction-limited field-of-view. We use spin-RESOLFT microscopy to\\nenable precision nanoscale mapping of magnetic field patterns with resolution\\ndown to ~20 nm, while employing a low power optical depletion beam. Moreover,\\nwe use a shallow NV to demonstrate the detection of proton nuclear magnetic\\nresonance (NMR) signals exterior to the diamond, with 50 nm lateral imaging\\nresolution and without degrading the proton NMR linewidth.\\n',\n",
       "  'Comment: main text: 5 pages, 4 figures. Supplementary Material: 5 pages, 8\\n  figures'],\n",
       " 'format': [],\n",
       " 'identifier': ['http://arxiv.org/abs/1610.02097',\n",
       "  'Optics Express Vol. 25, Issue 10, pp. 11048-11064 (2017)',\n",
       "  'doi:10.1364/OE.25.011048'],\n",
       " 'language': [],\n",
       " 'publisher': [],\n",
       " 'relation': [],\n",
       " 'rights': [],\n",
       " 'source': [],\n",
       " 'subject': ['Quantum Physics',\n",
       "  'Condensed Matter - Mesoscale and Nanoscale Physics'],\n",
       " 'title': ['Superresolution optical magnetic imaging and spectroscopy using\\n  individual electronic spins in diamond'],\n",
       " 'type': ['text']}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# metadata is element 1 and can be converted to python dictionary with getMap()\n",
    "metadata = record[1]\n",
    "metadata.getMap()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get all open access ids (optional)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total # of ids: 7845\n"
     ]
    }
   ],
   "source": [
    "# get list of PMCIDs\n",
    "\n",
    "from_date = '1901-01-01'\n",
    "from_date = '2017-11-22'\n",
    "from_datetime = datetime.datetime.strptime(from_date,\"%Y-%m-%d\")\n",
    "\n",
    "ids = []\n",
    "# allrecords = client.listRecords(metadataPrefix=metapre, from_=from_date)\n",
    "for record in client.listIdentifiers(metadataPrefix='oai_dc',from_=from_datetime):\n",
    "     \n",
    "    ids.append(record.identifier())\n",
    "    \n",
    "print('total # of ids:', len(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "672.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "totalpapers = 1678766 # total numbers of OA papers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### download all records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "percent done:  0.7692307692307693\n",
      "percent done:  1.1538461538461537\n",
      "percent done:  1.5384615384615385\n",
      "percent done:  1.9230769230769231\n",
      "percent done:  2.3076923076923075\n",
      "percent done:  2.6923076923076925\n",
      "percent done:  3.076923076923077\n"
     ]
    },
    {
     "ename": "ConnectionResetError",
     "evalue": "[WinError 10054] An existing connection was forcibly closed by the remote host",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mConnectionResetError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-46-09a8f59a1d91>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[0mi\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mrecord\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mclient\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlistRecords\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmetadataPrefix\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'oai_dc'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m \u001b[1;31m#     print(record)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\site-packages\\oaipmh\\client.py\u001b[0m in \u001b[0;36mResumptionListGenerator\u001b[1;34m(firstBatch, nextBatch)\u001b[0m\n\u001b[0;32m    363\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtoken\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    364\u001b[0m             \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 365\u001b[1;33m         \u001b[0mresult\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtoken\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnextBatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtoken\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    366\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    367\u001b[0m def retrieveFromUrlWaiting(request,\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\site-packages\\oaipmh\\client.py\u001b[0m in \u001b[0;36mnextBatch\u001b[1;34m(token)\u001b[0m\n\u001b[0;32m    192\u001b[0m             tree = self.makeRequestErrorHandling(\n\u001b[0;32m    193\u001b[0m                 \u001b[0mverb\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'ListRecords'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 194\u001b[1;33m                 resumptionToken=token)\n\u001b[0m\u001b[0;32m    195\u001b[0m             return self.buildRecords(\n\u001b[0;32m    196\u001b[0m                 \u001b[0mmetadata_prefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnamespaces\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\site-packages\\oaipmh\\client.py\u001b[0m in \u001b[0;36mmakeRequestErrorHandling\u001b[1;34m(self, **kw)\u001b[0m\n\u001b[0;32m    284\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    285\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mmakeRequestErrorHandling\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 286\u001b[1;33m         \u001b[0mxml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmakeRequest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    287\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    288\u001b[0m             \u001b[0mtree\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxml\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\site-packages\\oaipmh\\client.py\u001b[0m in \u001b[0;36mmakeRequest\u001b[1;34m(self, **kw)\u001b[0m\n\u001b[0;32m    344\u001b[0m                     self._base_url, data=binary_data, headers=headers)\n\u001b[0;32m    345\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 346\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mretrieveFromUrlWaiting\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    347\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    348\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mbuildHeader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mheader_node\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnamespaces\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\site-packages\\oaipmh\\client.py\u001b[0m in \u001b[0;36mretrieveFromUrlWaiting\u001b[1;34m(request, wait_max, wait_default)\u001b[0m\n\u001b[0;32m    371\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwait_max\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    372\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 373\u001b[1;33m             \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0murllib2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0murlopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    374\u001b[0m             \u001b[0mtext\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    375\u001b[0m             \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\urllib\\request.py\u001b[0m in \u001b[0;36murlopen\u001b[1;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[0;32m    159\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    160\u001b[0m         \u001b[0mopener\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_opener\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 161\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mopener\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    162\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    163\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0minstall_opener\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mopener\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\urllib\\request.py\u001b[0m in \u001b[0;36mopen\u001b[1;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[0;32m    462\u001b[0m             \u001b[0mreq\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    463\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 464\u001b[1;33m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_open\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    465\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    466\u001b[0m         \u001b[1;31m# post-process response\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\urllib\\request.py\u001b[0m in \u001b[0;36m_open\u001b[1;34m(self, req, data)\u001b[0m\n\u001b[0;32m    480\u001b[0m         \u001b[0mprotocol\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mreq\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    481\u001b[0m         result = self._call_chain(self.handle_open, protocol, protocol +\n\u001b[1;32m--> 482\u001b[1;33m                                   '_open', req)\n\u001b[0m\u001b[0;32m    483\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    484\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\urllib\\request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[1;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[0;32m    440\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    441\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 442\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    443\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    444\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\urllib\\request.py\u001b[0m in \u001b[0;36mhttp_open\u001b[1;34m(self, req)\u001b[0m\n\u001b[0;32m   1209\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1210\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mhttp_open\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1211\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdo_open\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhttp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mHTTPConnection\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1212\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1213\u001b[0m     \u001b[0mhttp_request\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mAbstractHTTPHandler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdo_request_\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\urllib\\request.py\u001b[0m in \u001b[0;36mdo_open\u001b[1;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[0;32m   1184\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;31m# timeout error\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1185\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mURLError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1186\u001b[1;33m             \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mh\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1187\u001b[0m         \u001b[1;32mexcept\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1188\u001b[0m             \u001b[0mh\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\http\\client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1225\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1226\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1227\u001b[1;33m             \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1228\u001b[0m             \u001b[1;32massert\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwill_close\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0m_UNKNOWN\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1229\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__state\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_CS_IDLE\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\http\\client.py\u001b[0m in \u001b[0;36mbegin\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    384\u001b[0m         \u001b[1;31m# read until we get a non-100 response\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    385\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 386\u001b[1;33m             \u001b[0mversion\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    387\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    388\u001b[0m                 \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\http\\client.py\u001b[0m in \u001b[0;36m_read_status\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    346\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    347\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 348\u001b[1;33m         \u001b[0mline\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"iso-8859-1\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    349\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mline\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    350\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"status line\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\user\\Anaconda3\\envs\\python34\\lib\\socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[1;34m(self, b)\u001b[0m\n\u001b[0;32m    376\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    377\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 378\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    379\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    380\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mConnectionResetError\u001b[0m: [WinError 10054] An existing connection was forcibly closed by the remote host"
     ]
    }
   ],
   "source": [
    "# number of records per file\n",
    "batchsize = 2500\n",
    "\n",
    "# last file number - set to 0 if complete downlaod\n",
    "j = 1\n",
    "\n",
    "metadata_all = []\n",
    "id_counter = 0\n",
    "i = 0\n",
    "\n",
    "for record in client.listRecords(metadataPrefix='oai_dc'):\n",
    "#     print(record)\n",
    "    \n",
    "    # get current record id/metadata\n",
    "    header = record[0]\n",
    "    id = header.identifier().split(\"oai:arXiv.org:\")[1]\n",
    "        \n",
    "    metadata_raw = record[1]\n",
    "    \n",
    "    if metadata_raw:\n",
    "        metadata = metadata_raw.getMap()\n",
    "    else: \n",
    "        metadata = []\n",
    "            \n",
    "    metadata['id'] = id\n",
    "        \n",
    "    # append record data\n",
    "    metadata_all.append(metadata)\n",
    "            \n",
    "    if i >= batchsize:\n",
    "        \n",
    "        with open(r\"metadata_arxiv_%04d.txt\" % j, 'w') as outfile:     \n",
    "            json.dump(metadata_all, outfile)\n",
    "        \n",
    "        # reset counter/variables\n",
    "        i = 0\n",
    "        metadata_all = []\n",
    "        # increase file counter\n",
    "        j += 1\n",
    "    \n",
    "        print(\"percent done: \", 100*(j*batchsize)/1.3e6)\n",
    "    # increase paper counts\n",
    "    i += 1\n",
    "\n",
    "# save last amount of metadata\n",
    "j += 1\n",
    "with open(r\"metadata_arxiv_%04d.txt\" % j, 'w') as outfile:     \n",
    "            json.dump(metadata_all, outfile)\n",
    "print(\"total records:\", j*batchsize + len(metadata_all))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>contributor</th>\n",
       "      <th>coverage</th>\n",
       "      <th>creator</th>\n",
       "      <th>date</th>\n",
       "      <th>description</th>\n",
       "      <th>format</th>\n",
       "      <th>id</th>\n",
       "      <th>identifier</th>\n",
       "      <th>language</th>\n",
       "      <th>publisher</th>\n",
       "      <th>relation</th>\n",
       "      <th>rights</th>\n",
       "      <th>source</th>\n",
       "      <th>subject</th>\n",
       "      <th>title</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Gu, Ying-Qiu]</td>\n",
       "      <td>[2007-08-22, 2017-11-27]</td>\n",
       "      <td>[  In this paper, we provide a procedure to so...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0708.2962</td>\n",
       "      <td>[http://arxiv.org/abs/0708.2962, Quant. Phys. ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Physics - General Physics]</td>\n",
       "      <td>[A Procedure to Solve the Eigen Solution to Di...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Achlioptas, Dimitris, Coja-Oghlan, Amin]</td>\n",
       "      <td>[2008-03-14, 2008-05-21]</td>\n",
       "      <td>[  For many random Constraint Satisfaction Pro...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0803.2122</td>\n",
       "      <td>[http://arxiv.org/abs/0803.2122, Proc. 49th FO...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Combinatorics, Mathematics - Pr...</td>\n",
       "      <td>[Algorithmic barriers from phase transitions]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Liu, Changli]</td>\n",
       "      <td>[2010-02-03, 2017-11-28]</td>\n",
       "      <td>[  Maxwell's equations seem overdetermined, wh...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1002.0892</td>\n",
       "      <td>[http://arxiv.org/abs/1002.0892, Physics and E...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Physics - General Physics, Physics - Classica...</td>\n",
       "      <td>[Explanation on Overdetermination of Maxwell's...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Coja-Oghlan, Amin]</td>\n",
       "      <td>[2010-07-08, 2014-04-25]</td>\n",
       "      <td>[  Let F be a uniformly distributed random k-S...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1007.1328</td>\n",
       "      <td>[http://arxiv.org/abs/1007.1328, Proc. 22nd SO...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Combinatorics, Computer Science...</td>\n",
       "      <td>[On belief propagation guided decimation for r...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Coja-Oghlan, Amin, Efthymiou, Charilaos]</td>\n",
       "      <td>[2010-07-08, 2013-04-09]</td>\n",
       "      <td>[  The independence number of a sparse random ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1007.1378</td>\n",
       "      <td>[http://arxiv.org/abs/1007.1378, Random Struct...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Discrete Mathematics]</td>\n",
       "      <td>[On independent sets in random graphs]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Coja-Oghlan, Amin, Pachon-Pinzon, Angelica Y.]</td>\n",
       "      <td>[2011-02-15]</td>\n",
       "      <td>[  Let F be a uniformly distributed random k-S...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1102.3145</td>\n",
       "      <td>[http://arxiv.org/abs/1102.3145, SIAM Journal ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Combinatorics, Computer Science...</td>\n",
       "      <td>[The decimation process in random k-SAT]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Visser, Matt]</td>\n",
       "      <td>[2011-03-29, 2011-04-03]</td>\n",
       "      <td>[  Horava gravity is a relatively recent (Jan ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1103.5587</td>\n",
       "      <td>[http://arxiv.org/abs/1103.5587, J.Phys.Conf.S...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Theory, General Relativ...</td>\n",
       "      <td>[Status of Horava gravity: A personal perspect...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Gautier, Eric, Pennec, Erwan Le]</td>\n",
       "      <td>[2011-06-17, 2017-11-28]</td>\n",
       "      <td>[  In the random coefficients binary choice mo...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1106.3503</td>\n",
       "      <td>[http://arxiv.org/abs/1106.3503]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Statistics Theory]</td>\n",
       "      <td>[Adaptive estimation in the nonparametric rand...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Griffeth, Stephen]</td>\n",
       "      <td>[2011-06-24, 2017-11-28]</td>\n",
       "      <td>[  We classify the irreducible unitary modules...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1106.5094</td>\n",
       "      <td>[http://arxiv.org/abs/1106.5094]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Representation Theory, Mathemat...</td>\n",
       "      <td>[Unitary representations of cyclotomic rationa...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Najafizadeh, Mojtaba, Saadat, Mehdi]</td>\n",
       "      <td>[2011-08-22, 2013-04-05]</td>\n",
       "      <td>[  We study the formulation of statistical mec...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1108.4273</td>\n",
       "      <td>[http://arxiv.org/abs/1108.4273, Chin.J.Phys. ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Theory, Condensed Matte...</td>\n",
       "      <td>[Thermodynamics of Classical Systems on Noncom...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Visser, Matt]</td>\n",
       "      <td>[2011-08-26, 2011-10-20]</td>\n",
       "      <td>[  Entropic forces have recently attracted con...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1108.5240</td>\n",
       "      <td>[http://arxiv.org/abs/1108.5240, JHEP 1110 (20...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Theory, General Relativ...</td>\n",
       "      <td>[Conservative entropic forces]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Abreu, Gabriel, Barcelo, Carlos, Visser, Matt]</td>\n",
       "      <td>[2011-09-13, 2011-12-08]</td>\n",
       "      <td>[  In a pair of recent articles [PRL 105 (2010...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1109.2710</td>\n",
       "      <td>[http://arxiv.org/abs/1109.2710, JHEP 1112 (20...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[General Relativity and Quantum Cosmology, Hig...</td>\n",
       "      <td>[Entropy bounds in terms of the w parameter]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Baccetti, Valentina, Tate, Kyle, Visser, Matt]</td>\n",
       "      <td>[2011-11-27, 2011-12-04]</td>\n",
       "      <td>[  Recent tentative experimental indications, ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1111.6340</td>\n",
       "      <td>[http://arxiv.org/abs/1111.6340, JHEP 1203 (20...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Phenomenology, General ...</td>\n",
       "      <td>[Lorentz violating kinematics: Threshold theor...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Moussard, Delphine]</td>\n",
       "      <td>[2012-07-09, 2013-03-26]</td>\n",
       "      <td>[  Null Lagrangian-preserving surgeries are a ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1207.1954</td>\n",
       "      <td>[http://arxiv.org/abs/1207.1954, Bulletin de l...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Algebraic Topology, Mathematics...</td>\n",
       "      <td>[Rational Blanchfield forms, S-equivalence, an...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Coja-Oghlan, Amin, Panagiotou, Konstantinos]</td>\n",
       "      <td>[2012-12-07]</td>\n",
       "      <td>[  Random $k$-SAT is the single most intensely...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1212.1682</td>\n",
       "      <td>[http://arxiv.org/abs/1212.1682, Proc. 45 th S...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Combinatorics, Computer Science...</td>\n",
       "      <td>[Going after the k-SAT Threshold]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Angella, Daniele, Kasuya, Hisashi]</td>\n",
       "      <td>[2012-12-22, 2017-04-18]</td>\n",
       "      <td>[  We study conditions under which sub-complex...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1212.5708</td>\n",
       "      <td>[http://arxiv.org/abs/1212.5708, Ann. Global A...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Differential Geometry, Mathemat...</td>\n",
       "      <td>[Bott-Chern cohomology of solvmanifolds]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Alonso-Izquierdo, Alberto, Mateos-Guilarte, J...</td>\n",
       "      <td>[2013-03-13]</td>\n",
       "      <td>[  In this paper we propose a generalization o...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1303.3138</td>\n",
       "      <td>[http://arxiv.org/abs/1303.3138, Il Nuovo Cime...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Theory]</td>\n",
       "      <td>[Gilkey-de Witt heat kernel expansion and zero...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Baccetti, Valentina, Visser, Matt]</td>\n",
       "      <td>[2013-03-13, 2013-12-05]</td>\n",
       "      <td>[  Jacobson's thermodynamic derivation of the ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1303.3185</td>\n",
       "      <td>[http://arxiv.org/abs/1303.3185, Class.Quant.G...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[General Relativity and Quantum Cosmology, Hig...</td>\n",
       "      <td>[Clausius entropy for arbitrary bifurcate null...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Shemyakova, Ekaterina]</td>\n",
       "      <td>[2013-04-26, 2017-11-27]</td>\n",
       "      <td>[  Darboux transformations are non-group type ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1304.7063</td>\n",
       "      <td>[http://arxiv.org/abs/1304.7063]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematical Physics]</td>\n",
       "      <td>[Classification of Darboux transformations for...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Coja-Oghlan, Amin]</td>\n",
       "      <td>[2013-05-01, 2013-05-19]</td>\n",
       "      <td>[  Let $G(n,m)$ be the random graph on $n$ ver...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1305.0177</td>\n",
       "      <td>[http://arxiv.org/abs/1305.0177, Electronic Jo...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Combinatorics, Computer Science...</td>\n",
       "      <td>[Upper-bounding the k-colorability threshold b...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Prashanth, L. A., Korda, Nathaniel, Munos, Rémi]</td>\n",
       "      <td>[2013-06-11, 2017-11-28]</td>\n",
       "      <td>[  We propose a stochastic approximation (SA) ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1306.2557</td>\n",
       "      <td>[http://arxiv.org/abs/1306.2557]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Learning, Statistics - Mac...</td>\n",
       "      <td>[Stochastic approximation for speeding up LSTD...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Alonso-Izquierdo, Alberto, Guilarte, Juan Mat...</td>\n",
       "      <td>[2013-07-02, 2014-01-17]</td>\n",
       "      <td>[  In this paper quantum effects are investiga...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1307.0740</td>\n",
       "      <td>[http://arxiv.org/abs/1307.0740, JHEP01 (2014)...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Theory]</td>\n",
       "      <td>[Quantum-induced interactions in the moduli sp...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Daws, Matthew, Spronk, Nico]</td>\n",
       "      <td>[2013-08-05, 2017-11-27]</td>\n",
       "      <td>[  We prove two theorems about convolution ope...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1308.1073</td>\n",
       "      <td>[http://arxiv.org/abs/1308.1073]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Functional Analysis, Mathematic...</td>\n",
       "      <td>[On convoluters on $L^p$-spaces]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Zhao, Zhi-Dan, Gao, Ya-Chun, Cai, Shi-Min]</td>\n",
       "      <td>[2013-08-22, 2015-03-24]</td>\n",
       "      <td>[  A mass of traces of human activities show d...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1308.4847</td>\n",
       "      <td>[http://arxiv.org/abs/1308.4847, Physica A 461...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Physics - Physics and Society, Computer Scien...</td>\n",
       "      <td>[The dynamic pattern of human attention]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Parusinski, Adam, Rainer, Armin]</td>\n",
       "      <td>[2013-09-09, 2015-12-18]</td>\n",
       "      <td>[  We give a new self-contained proof of Brons...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1309.2150</td>\n",
       "      <td>[http://arxiv.org/abs/1309.2150, J. Hyperbolic...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Classical Analysis and ODEs, 26...</td>\n",
       "      <td>[A new proof of Bronshtein's theorem]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mertsch, Philipp, Sarkar, Subir]</td>\n",
       "      <td>[2014-02-04, 2014-08-05]</td>\n",
       "      <td>[  We revisit the model proposed earlier to ac...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1402.0855</td>\n",
       "      <td>[http://arxiv.org/abs/1402.0855, Phys. Rev. D ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Astrophysics - High Energy Astrophysical Phen...</td>\n",
       "      <td>[AMS-02 data confronts acceleration of cosmic ...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Aizenbud, Avraham, Gourevitch, Dmitry, Krötz,...</td>\n",
       "      <td>[2014-03-24, 2016-05-05]</td>\n",
       "      <td>[  Let $H$ be a real algebraic group acting eq...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1403.5917</td>\n",
       "      <td>[http://arxiv.org/abs/1403.5917, Math. Z. 286 ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Representation Theory, 20G05, 2...</td>\n",
       "      <td>[Hausdorffness for Lie algebra homology of Sch...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Bapst, Victor, Coja-Oghlan, Amin, Hetterich, ...</td>\n",
       "      <td>[2014-04-19]</td>\n",
       "      <td>[  Based on a non-rigorous formalism called th...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1404.5513</td>\n",
       "      <td>[http://arxiv.org/abs/1404.5513, Communication...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Discrete Mathematics, Math...</td>\n",
       "      <td>[The condensation phase transition in random g...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Parusinski, Adam, Rainer, Armin]</td>\n",
       "      <td>[2014-06-10, 2015-10-03]</td>\n",
       "      <td>[  Let $\\rho : G \\rightarrow \\operatorname{O}(...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1406.2485</td>\n",
       "      <td>[http://arxiv.org/abs/1406.2485, Transformatio...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Differential Geometry, Mathemat...</td>\n",
       "      <td>[Lifting differentiable curves from orbit spaces]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Kohls, Martin, Sezer, Müfit]</td>\n",
       "      <td>[2014-06-24]</td>\n",
       "      <td>[  For a finite dimensional representation $V$...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1406.6299</td>\n",
       "      <td>[http://arxiv.org/abs/1406.6299, Communication...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Commutative Algebra, Mathematic...</td>\n",
       "      <td>[Degree of reductivity of a modular representa...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1014</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Goyal, Manu, Reeves, Neil D., Davison, Adrian...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Globally, in 2016, one out of eleven adults...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10448</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10448]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Computer Vision and Patter...</td>\n",
       "      <td>[DFUNet: Convolutional Neural Networks for Dia...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1015</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Goyal, Manu, Yap, Moi Hoon]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Early detection of skin cancer, particularl...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10449</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10449]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Computer Vision and Patter...</td>\n",
       "      <td>[Multi-class Semantic Segmentation of Skin Les...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1016</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Cigoli, Alan S., Everaert, Tomas, Gran, Marino]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Given an exact category $\\mathcal{C}$, it i...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10450</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10450]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Category Theory, Mathematics - ...</td>\n",
       "      <td>[A relative monotone-light factorisation syste...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1017</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Browning, Tim, Sawin, W.]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We develop a geometric version of the circl...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10451</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10451]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Algebraic Geometry, Mathematics...</td>\n",
       "      <td>[A geometric version of the circle method]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1018</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Gillman, Edward, Rajantie, Arttu]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  The Kibble Zurek mechanism in a relativisti...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10452</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10452]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Quantum Physics, High Energy Physics - Lattice]</td>\n",
       "      <td>[The Kibble Zurek Mechanism of Topological Def...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1019</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Strickland, Mark, Fainekos, Georgios, Amor, H...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  In this paper, we investigate a predictive ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10453</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10453]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Robotics]</td>\n",
       "      <td>[Deep Predictive Models for Collision Risk Ass...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1020</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Motlagh, . Azar Oliaei, Wu, Jhih-Sheng, Apalk...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We predict a possibility to induce a signif...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10454</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10454]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Condensed Matter - Mesoscale and Nanoscale Ph...</td>\n",
       "      <td>[Femtosecond valley polarization and topologic...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1021</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Fong, Brendan, Spivak, David I., Tuyéras, Rémy]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  A supervised learning algorithm searches ov...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10455</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10455]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Category Theory, Computer Scien...</td>\n",
       "      <td>[Backprop as Functor: A compositional perspect...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Jin, Chi, Netrapalli, Praneeth, Jordan, Micha...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Nesterov's accelerated gradient descent (AG...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10456</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10456]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Learning, Mathematics - Op...</td>\n",
       "      <td>[Accelerated Gradient Descent Escapes Saddle P...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1023</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Moia, Davide, Giovannitti, Alexander, Szumska...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We report a neutral salt water based batter...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10457</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10457]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Physics - Applied Physics, Condensed Matter -...</td>\n",
       "      <td>[A salt water battery with high stability and ...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1024</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Clesse, Sebastien, García-Bellido, Juan]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  The detection by Advanced LIGO/VIRGO of bla...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10458</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10458]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Astrophysics - Cosmology and Nongalactic Astr...</td>\n",
       "      <td>[Seven Hints for Primordial Black Hole Dark Ma...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1025</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Zhuang, Quntao, Zhang, Zheshen, Shapiro, Jeff...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Distributed quantum sensing uses quantum co...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10459</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10459]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Quantum Physics]</td>\n",
       "      <td>[Distributed Quantum Sensing Using Continuous-...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1026</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Babbush, Ryan, Berry, Dominic, Kieferová, Már...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We present three techniques for reducing th...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10460</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10460]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Quantum Physics]</td>\n",
       "      <td>[Improved Techniques for Preparing Eigenstates...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1027</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Kravchuk, Volodymyr P., Sheka, Denis D., Rößl...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Properties of magnon modes localized on a f...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10461</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10461]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Condensed Matter - Strongly Correlated Electr...</td>\n",
       "      <td>[Spin eigen-excitations of the magnetic skyrmi...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1028</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Dutil, Francis, Gulcehre, Caglar, Trischler, ...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We investigate the integration of a plannin...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10462</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10462]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Learning, Statistics - Mac...</td>\n",
       "      <td>[Plan, Attend, Generate: Planning for Sequence...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1029</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mastrantonio, Gianluca]</td>\n",
       "      <td>[2017-11-21]</td>\n",
       "      <td>[  The contribution of this work is the introd...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10463</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10463]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Statistics - Methodology]</td>\n",
       "      <td>[The joint projected normal and skew-normal: a...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1030</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Abdelkader, Ibrahim, El-Sonbaty, Yasser, El-H...</td>\n",
       "      <td>[2017-11-01]</td>\n",
       "      <td>[  Advances in semiconductor manufacturing pro...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10464</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10464, Internationa...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Computer Vision and Patter...</td>\n",
       "      <td>[Openmv: A Python powered, extensible machine ...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1031</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Duarte, Cristhiano, Amaral, Barbara]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Contextuality has been identified as a pote...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10465</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10465]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Quantum Physics]</td>\n",
       "      <td>[Resource theory of contextuality for arbitrar...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1032</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Bednorz, Adam, Bednorz, Witold]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Sphere eversions have been described so far...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10466</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10466]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Geometric Topology, Mathematica...</td>\n",
       "      <td>[Analytic sphere eversion with minimum of topo...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1033</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Ma, Cong, Wang, Kaizheng, Chi, Yuejie, Chen, ...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Recent years have seen a flurry of activiti...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10467</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10467]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Computer Science - Learning, Computer Science...</td>\n",
       "      <td>[Implicit Regularization in Nonconvex Statisti...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1034</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Belton, Alexander, Guillot, Dominique, Khare,...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We prove that the only entrywise transforms...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10468</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10468]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Functional Analysis, Mathematic...</td>\n",
       "      <td>[Total-positivity preservers]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1035</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mosca, S., Parisi, M., Ricciardi, I., Leo, F....</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  Continuously pumped passive nonlinear cavit...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10469</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10469]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Physics - Optics]</td>\n",
       "      <td>[Frequency comb generation in a continuously p...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1036</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Even-Zohar, Chaim]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  The study of knots and links from a probabi...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10470</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10470, J Appl. and ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Geometric Topology, Mathematics...</td>\n",
       "      <td>[Models of Random Knots]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1037</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Huang, Zhoushen, Balatsky, Alexander V.]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We show that the Hilbert space spanned by a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10471</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10471]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Condensed Matter - Statistical Mechanics, Con...</td>\n",
       "      <td>[Complexity and geometry of quantum state mani...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1038</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Lin, Chunshan, Quintin, Jerome, Brandenberger...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We consider a modified gravity model with a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10472</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10472]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Theory, Astrophysics - ...</td>\n",
       "      <td>[Massive gravity and the suppression of anisot...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1039</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Yin, Shuai, Jian, Shao-Kai, Yao, Hong]</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We explore the universal properties of a ch...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10473</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10473]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Condensed Matter - Strongly Correlated Electr...</td>\n",
       "      <td>[Chiral tricritical point: a new universality ...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Bena, Iosif, Giusto, Stefano, Martinec, Emil ...</td>\n",
       "      <td>[2017-11-28]</td>\n",
       "      <td>[  We construct an infinite family of smooth a...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1711.10474</td>\n",
       "      <td>[http://arxiv.org/abs/1711.10474]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Theory, General Relativ...</td>\n",
       "      <td>[Asymptotically-flat supergravity solutions de...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Ballesteros, Angel, Bruno, N. Rossano, Herran...</td>\n",
       "      <td>[2004-01-30, 2017-08-06]</td>\n",
       "      <td>[  The $\\kappa$-deformation of the (2+1)D anti...</td>\n",
       "      <td>[]</td>\n",
       "      <td>hep-th/0401244</td>\n",
       "      <td>[http://arxiv.org/abs/hep-th/0401244, Advances...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Theory]</td>\n",
       "      <td>[Non-commutative relativistic spacetimes and w...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1042</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Gu, Ying-Qiu]</td>\n",
       "      <td>[2006-11-19, 2009-06-06]</td>\n",
       "      <td>[  In this paper, we solve the eigen solutions...</td>\n",
       "      <td>[]</td>\n",
       "      <td>hep-th/0611210</td>\n",
       "      <td>[http://arxiv.org/abs/hep-th/0611210, Quant. P...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[High Energy Physics - Theory, General Relativ...</td>\n",
       "      <td>[The Characteristic Functions and Their Typica...</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1043</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Gabber, Ofer, Ramero, Lorenzo]</td>\n",
       "      <td>[2004-09-29, 2017-11-28]</td>\n",
       "      <td>[  This is release 7 of our project, aiming to...</td>\n",
       "      <td>[]</td>\n",
       "      <td>math/0409584</td>\n",
       "      <td>[http://arxiv.org/abs/math/0409584]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mathematics - Algebraic Geometry, Mathematics...</td>\n",
       "      <td>[Foundations for almost ring theory -- Release 7]</td>\n",
       "      <td>[text]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1044 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     contributor coverage                                            creator  \\\n",
       "0             []       []                                     [Gu, Ying-Qiu]   \n",
       "1             []       []          [Achlioptas, Dimitris, Coja-Oghlan, Amin]   \n",
       "2             []       []                                     [Liu, Changli]   \n",
       "3             []       []                                [Coja-Oghlan, Amin]   \n",
       "4             []       []          [Coja-Oghlan, Amin, Efthymiou, Charilaos]   \n",
       "5             []       []    [Coja-Oghlan, Amin, Pachon-Pinzon, Angelica Y.]   \n",
       "6             []       []                                     [Visser, Matt]   \n",
       "7             []       []                  [Gautier, Eric, Pennec, Erwan Le]   \n",
       "8             []       []                                [Griffeth, Stephen]   \n",
       "9             []       []              [Najafizadeh, Mojtaba, Saadat, Mehdi]   \n",
       "10            []       []                                     [Visser, Matt]   \n",
       "11            []       []    [Abreu, Gabriel, Barcelo, Carlos, Visser, Matt]   \n",
       "12            []       []    [Baccetti, Valentina, Tate, Kyle, Visser, Matt]   \n",
       "13            []       []                               [Moussard, Delphine]   \n",
       "14            []       []      [Coja-Oghlan, Amin, Panagiotou, Konstantinos]   \n",
       "15            []       []                [Angella, Daniele, Kasuya, Hisashi]   \n",
       "16            []       []  [Alonso-Izquierdo, Alberto, Mateos-Guilarte, J...   \n",
       "17            []       []                [Baccetti, Valentina, Visser, Matt]   \n",
       "18            []       []                            [Shemyakova, Ekaterina]   \n",
       "19            []       []                                [Coja-Oghlan, Amin]   \n",
       "20            []       []  [Prashanth, L. A., Korda, Nathaniel, Munos, Rémi]   \n",
       "21            []       []  [Alonso-Izquierdo, Alberto, Guilarte, Juan Mat...   \n",
       "22            []       []                      [Daws, Matthew, Spronk, Nico]   \n",
       "23            []       []        [Zhao, Zhi-Dan, Gao, Ya-Chun, Cai, Shi-Min]   \n",
       "24            []       []                  [Parusinski, Adam, Rainer, Armin]   \n",
       "25            []       []                  [Mertsch, Philipp, Sarkar, Subir]   \n",
       "26            []       []  [Aizenbud, Avraham, Gourevitch, Dmitry, Krötz,...   \n",
       "27            []       []  [Bapst, Victor, Coja-Oghlan, Amin, Hetterich, ...   \n",
       "28            []       []                  [Parusinski, Adam, Rainer, Armin]   \n",
       "29            []       []                      [Kohls, Martin, Sezer, Müfit]   \n",
       "...          ...      ...                                                ...   \n",
       "1014          []       []  [Goyal, Manu, Reeves, Neil D., Davison, Adrian...   \n",
       "1015          []       []                       [Goyal, Manu, Yap, Moi Hoon]   \n",
       "1016          []       []   [Cigoli, Alan S., Everaert, Tomas, Gran, Marino]   \n",
       "1017          []       []                         [Browning, Tim, Sawin, W.]   \n",
       "1018          []       []                 [Gillman, Edward, Rajantie, Arttu]   \n",
       "1019          []       []  [Strickland, Mark, Fainekos, Georgios, Amor, H...   \n",
       "1020          []       []  [Motlagh, . Azar Oliaei, Wu, Jhih-Sheng, Apalk...   \n",
       "1021          []       []   [Fong, Brendan, Spivak, David I., Tuyéras, Rémy]   \n",
       "1022          []       []  [Jin, Chi, Netrapalli, Praneeth, Jordan, Micha...   \n",
       "1023          []       []  [Moia, Davide, Giovannitti, Alexander, Szumska...   \n",
       "1024          []       []          [Clesse, Sebastien, García-Bellido, Juan]   \n",
       "1025          []       []  [Zhuang, Quntao, Zhang, Zheshen, Shapiro, Jeff...   \n",
       "1026          []       []  [Babbush, Ryan, Berry, Dominic, Kieferová, Már...   \n",
       "1027          []       []  [Kravchuk, Volodymyr P., Sheka, Denis D., Rößl...   \n",
       "1028          []       []  [Dutil, Francis, Gulcehre, Caglar, Trischler, ...   \n",
       "1029          []       []                           [Mastrantonio, Gianluca]   \n",
       "1030          []       []  [Abdelkader, Ibrahim, El-Sonbaty, Yasser, El-H...   \n",
       "1031          []       []              [Duarte, Cristhiano, Amaral, Barbara]   \n",
       "1032          []       []                   [Bednorz, Adam, Bednorz, Witold]   \n",
       "1033          []       []  [Ma, Cong, Wang, Kaizheng, Chi, Yuejie, Chen, ...   \n",
       "1034          []       []  [Belton, Alexander, Guillot, Dominique, Khare,...   \n",
       "1035          []       []  [Mosca, S., Parisi, M., Ricciardi, I., Leo, F....   \n",
       "1036          []       []                                [Even-Zohar, Chaim]   \n",
       "1037          []       []          [Huang, Zhoushen, Balatsky, Alexander V.]   \n",
       "1038          []       []  [Lin, Chunshan, Quintin, Jerome, Brandenberger...   \n",
       "1039          []       []            [Yin, Shuai, Jian, Shao-Kai, Yao, Hong]   \n",
       "1040          []       []  [Bena, Iosif, Giusto, Stefano, Martinec, Emil ...   \n",
       "1041          []       []  [Ballesteros, Angel, Bruno, N. Rossano, Herran...   \n",
       "1042          []       []                                     [Gu, Ying-Qiu]   \n",
       "1043          []       []                    [Gabber, Ofer, Ramero, Lorenzo]   \n",
       "\n",
       "                          date  \\\n",
       "0     [2007-08-22, 2017-11-27]   \n",
       "1     [2008-03-14, 2008-05-21]   \n",
       "2     [2010-02-03, 2017-11-28]   \n",
       "3     [2010-07-08, 2014-04-25]   \n",
       "4     [2010-07-08, 2013-04-09]   \n",
       "5                 [2011-02-15]   \n",
       "6     [2011-03-29, 2011-04-03]   \n",
       "7     [2011-06-17, 2017-11-28]   \n",
       "8     [2011-06-24, 2017-11-28]   \n",
       "9     [2011-08-22, 2013-04-05]   \n",
       "10    [2011-08-26, 2011-10-20]   \n",
       "11    [2011-09-13, 2011-12-08]   \n",
       "12    [2011-11-27, 2011-12-04]   \n",
       "13    [2012-07-09, 2013-03-26]   \n",
       "14                [2012-12-07]   \n",
       "15    [2012-12-22, 2017-04-18]   \n",
       "16                [2013-03-13]   \n",
       "17    [2013-03-13, 2013-12-05]   \n",
       "18    [2013-04-26, 2017-11-27]   \n",
       "19    [2013-05-01, 2013-05-19]   \n",
       "20    [2013-06-11, 2017-11-28]   \n",
       "21    [2013-07-02, 2014-01-17]   \n",
       "22    [2013-08-05, 2017-11-27]   \n",
       "23    [2013-08-22, 2015-03-24]   \n",
       "24    [2013-09-09, 2015-12-18]   \n",
       "25    [2014-02-04, 2014-08-05]   \n",
       "26    [2014-03-24, 2016-05-05]   \n",
       "27                [2014-04-19]   \n",
       "28    [2014-06-10, 2015-10-03]   \n",
       "29                [2014-06-24]   \n",
       "...                        ...   \n",
       "1014              [2017-11-28]   \n",
       "1015              [2017-11-28]   \n",
       "1016              [2017-11-28]   \n",
       "1017              [2017-11-28]   \n",
       "1018              [2017-11-28]   \n",
       "1019              [2017-11-28]   \n",
       "1020              [2017-11-28]   \n",
       "1021              [2017-11-28]   \n",
       "1022              [2017-11-28]   \n",
       "1023              [2017-11-28]   \n",
       "1024              [2017-11-28]   \n",
       "1025              [2017-11-28]   \n",
       "1026              [2017-11-28]   \n",
       "1027              [2017-11-28]   \n",
       "1028              [2017-11-28]   \n",
       "1029              [2017-11-21]   \n",
       "1030              [2017-11-01]   \n",
       "1031              [2017-11-28]   \n",
       "1032              [2017-11-28]   \n",
       "1033              [2017-11-28]   \n",
       "1034              [2017-11-28]   \n",
       "1035              [2017-11-28]   \n",
       "1036              [2017-11-28]   \n",
       "1037              [2017-11-28]   \n",
       "1038              [2017-11-28]   \n",
       "1039              [2017-11-28]   \n",
       "1040              [2017-11-28]   \n",
       "1041  [2004-01-30, 2017-08-06]   \n",
       "1042  [2006-11-19, 2009-06-06]   \n",
       "1043  [2004-09-29, 2017-11-28]   \n",
       "\n",
       "                                            description format  \\\n",
       "0     [  In this paper, we provide a procedure to so...     []   \n",
       "1     [  For many random Constraint Satisfaction Pro...     []   \n",
       "2     [  Maxwell's equations seem overdetermined, wh...     []   \n",
       "3     [  Let F be a uniformly distributed random k-S...     []   \n",
       "4     [  The independence number of a sparse random ...     []   \n",
       "5     [  Let F be a uniformly distributed random k-S...     []   \n",
       "6     [  Horava gravity is a relatively recent (Jan ...     []   \n",
       "7     [  In the random coefficients binary choice mo...     []   \n",
       "8     [  We classify the irreducible unitary modules...     []   \n",
       "9     [  We study the formulation of statistical mec...     []   \n",
       "10    [  Entropic forces have recently attracted con...     []   \n",
       "11    [  In a pair of recent articles [PRL 105 (2010...     []   \n",
       "12    [  Recent tentative experimental indications, ...     []   \n",
       "13    [  Null Lagrangian-preserving surgeries are a ...     []   \n",
       "14    [  Random $k$-SAT is the single most intensely...     []   \n",
       "15    [  We study conditions under which sub-complex...     []   \n",
       "16    [  In this paper we propose a generalization o...     []   \n",
       "17    [  Jacobson's thermodynamic derivation of the ...     []   \n",
       "18    [  Darboux transformations are non-group type ...     []   \n",
       "19    [  Let $G(n,m)$ be the random graph on $n$ ver...     []   \n",
       "20    [  We propose a stochastic approximation (SA) ...     []   \n",
       "21    [  In this paper quantum effects are investiga...     []   \n",
       "22    [  We prove two theorems about convolution ope...     []   \n",
       "23    [  A mass of traces of human activities show d...     []   \n",
       "24    [  We give a new self-contained proof of Brons...     []   \n",
       "25    [  We revisit the model proposed earlier to ac...     []   \n",
       "26    [  Let $H$ be a real algebraic group acting eq...     []   \n",
       "27    [  Based on a non-rigorous formalism called th...     []   \n",
       "28    [  Let $\\rho : G \\rightarrow \\operatorname{O}(...     []   \n",
       "29    [  For a finite dimensional representation $V$...     []   \n",
       "...                                                 ...    ...   \n",
       "1014  [  Globally, in 2016, one out of eleven adults...     []   \n",
       "1015  [  Early detection of skin cancer, particularl...     []   \n",
       "1016  [  Given an exact category $\\mathcal{C}$, it i...     []   \n",
       "1017  [  We develop a geometric version of the circl...     []   \n",
       "1018  [  The Kibble Zurek mechanism in a relativisti...     []   \n",
       "1019  [  In this paper, we investigate a predictive ...     []   \n",
       "1020  [  We predict a possibility to induce a signif...     []   \n",
       "1021  [  A supervised learning algorithm searches ov...     []   \n",
       "1022  [  Nesterov's accelerated gradient descent (AG...     []   \n",
       "1023  [  We report a neutral salt water based batter...     []   \n",
       "1024  [  The detection by Advanced LIGO/VIRGO of bla...     []   \n",
       "1025  [  Distributed quantum sensing uses quantum co...     []   \n",
       "1026  [  We present three techniques for reducing th...     []   \n",
       "1027  [  Properties of magnon modes localized on a f...     []   \n",
       "1028  [  We investigate the integration of a plannin...     []   \n",
       "1029  [  The contribution of this work is the introd...     []   \n",
       "1030  [  Advances in semiconductor manufacturing pro...     []   \n",
       "1031  [  Contextuality has been identified as a pote...     []   \n",
       "1032  [  Sphere eversions have been described so far...     []   \n",
       "1033  [  Recent years have seen a flurry of activiti...     []   \n",
       "1034  [  We prove that the only entrywise transforms...     []   \n",
       "1035  [  Continuously pumped passive nonlinear cavit...     []   \n",
       "1036  [  The study of knots and links from a probabi...     []   \n",
       "1037  [  We show that the Hilbert space spanned by a...     []   \n",
       "1038  [  We consider a modified gravity model with a...     []   \n",
       "1039  [  We explore the universal properties of a ch...     []   \n",
       "1040  [  We construct an infinite family of smooth a...     []   \n",
       "1041  [  The $\\kappa$-deformation of the (2+1)D anti...     []   \n",
       "1042  [  In this paper, we solve the eigen solutions...     []   \n",
       "1043  [  This is release 7 of our project, aiming to...     []   \n",
       "\n",
       "                  id                                         identifier  \\\n",
       "0          0708.2962  [http://arxiv.org/abs/0708.2962, Quant. Phys. ...   \n",
       "1          0803.2122  [http://arxiv.org/abs/0803.2122, Proc. 49th FO...   \n",
       "2          1002.0892  [http://arxiv.org/abs/1002.0892, Physics and E...   \n",
       "3          1007.1328  [http://arxiv.org/abs/1007.1328, Proc. 22nd SO...   \n",
       "4          1007.1378  [http://arxiv.org/abs/1007.1378, Random Struct...   \n",
       "5          1102.3145  [http://arxiv.org/abs/1102.3145, SIAM Journal ...   \n",
       "6          1103.5587  [http://arxiv.org/abs/1103.5587, J.Phys.Conf.S...   \n",
       "7          1106.3503                   [http://arxiv.org/abs/1106.3503]   \n",
       "8          1106.5094                   [http://arxiv.org/abs/1106.5094]   \n",
       "9          1108.4273  [http://arxiv.org/abs/1108.4273, Chin.J.Phys. ...   \n",
       "10         1108.5240  [http://arxiv.org/abs/1108.5240, JHEP 1110 (20...   \n",
       "11         1109.2710  [http://arxiv.org/abs/1109.2710, JHEP 1112 (20...   \n",
       "12         1111.6340  [http://arxiv.org/abs/1111.6340, JHEP 1203 (20...   \n",
       "13         1207.1954  [http://arxiv.org/abs/1207.1954, Bulletin de l...   \n",
       "14         1212.1682  [http://arxiv.org/abs/1212.1682, Proc. 45 th S...   \n",
       "15         1212.5708  [http://arxiv.org/abs/1212.5708, Ann. Global A...   \n",
       "16         1303.3138  [http://arxiv.org/abs/1303.3138, Il Nuovo Cime...   \n",
       "17         1303.3185  [http://arxiv.org/abs/1303.3185, Class.Quant.G...   \n",
       "18         1304.7063                   [http://arxiv.org/abs/1304.7063]   \n",
       "19         1305.0177  [http://arxiv.org/abs/1305.0177, Electronic Jo...   \n",
       "20         1306.2557                   [http://arxiv.org/abs/1306.2557]   \n",
       "21         1307.0740  [http://arxiv.org/abs/1307.0740, JHEP01 (2014)...   \n",
       "22         1308.1073                   [http://arxiv.org/abs/1308.1073]   \n",
       "23         1308.4847  [http://arxiv.org/abs/1308.4847, Physica A 461...   \n",
       "24         1309.2150  [http://arxiv.org/abs/1309.2150, J. Hyperbolic...   \n",
       "25         1402.0855  [http://arxiv.org/abs/1402.0855, Phys. Rev. D ...   \n",
       "26         1403.5917  [http://arxiv.org/abs/1403.5917, Math. Z. 286 ...   \n",
       "27         1404.5513  [http://arxiv.org/abs/1404.5513, Communication...   \n",
       "28         1406.2485  [http://arxiv.org/abs/1406.2485, Transformatio...   \n",
       "29         1406.6299  [http://arxiv.org/abs/1406.6299, Communication...   \n",
       "...              ...                                                ...   \n",
       "1014      1711.10448                  [http://arxiv.org/abs/1711.10448]   \n",
       "1015      1711.10449                  [http://arxiv.org/abs/1711.10449]   \n",
       "1016      1711.10450                  [http://arxiv.org/abs/1711.10450]   \n",
       "1017      1711.10451                  [http://arxiv.org/abs/1711.10451]   \n",
       "1018      1711.10452                  [http://arxiv.org/abs/1711.10452]   \n",
       "1019      1711.10453                  [http://arxiv.org/abs/1711.10453]   \n",
       "1020      1711.10454                  [http://arxiv.org/abs/1711.10454]   \n",
       "1021      1711.10455                  [http://arxiv.org/abs/1711.10455]   \n",
       "1022      1711.10456                  [http://arxiv.org/abs/1711.10456]   \n",
       "1023      1711.10457                  [http://arxiv.org/abs/1711.10457]   \n",
       "1024      1711.10458                  [http://arxiv.org/abs/1711.10458]   \n",
       "1025      1711.10459                  [http://arxiv.org/abs/1711.10459]   \n",
       "1026      1711.10460                  [http://arxiv.org/abs/1711.10460]   \n",
       "1027      1711.10461                  [http://arxiv.org/abs/1711.10461]   \n",
       "1028      1711.10462                  [http://arxiv.org/abs/1711.10462]   \n",
       "1029      1711.10463                  [http://arxiv.org/abs/1711.10463]   \n",
       "1030      1711.10464  [http://arxiv.org/abs/1711.10464, Internationa...   \n",
       "1031      1711.10465                  [http://arxiv.org/abs/1711.10465]   \n",
       "1032      1711.10466                  [http://arxiv.org/abs/1711.10466]   \n",
       "1033      1711.10467                  [http://arxiv.org/abs/1711.10467]   \n",
       "1034      1711.10468                  [http://arxiv.org/abs/1711.10468]   \n",
       "1035      1711.10469                  [http://arxiv.org/abs/1711.10469]   \n",
       "1036      1711.10470  [http://arxiv.org/abs/1711.10470, J Appl. and ...   \n",
       "1037      1711.10471                  [http://arxiv.org/abs/1711.10471]   \n",
       "1038      1711.10472                  [http://arxiv.org/abs/1711.10472]   \n",
       "1039      1711.10473                  [http://arxiv.org/abs/1711.10473]   \n",
       "1040      1711.10474                  [http://arxiv.org/abs/1711.10474]   \n",
       "1041  hep-th/0401244  [http://arxiv.org/abs/hep-th/0401244, Advances...   \n",
       "1042  hep-th/0611210  [http://arxiv.org/abs/hep-th/0611210, Quant. P...   \n",
       "1043    math/0409584                [http://arxiv.org/abs/math/0409584]   \n",
       "\n",
       "     language publisher relation rights source  \\\n",
       "0          []        []       []     []     []   \n",
       "1          []        []       []     []     []   \n",
       "2          []        []       []     []     []   \n",
       "3          []        []       []     []     []   \n",
       "4          []        []       []     []     []   \n",
       "5          []        []       []     []     []   \n",
       "6          []        []       []     []     []   \n",
       "7          []        []       []     []     []   \n",
       "8          []        []       []     []     []   \n",
       "9          []        []       []     []     []   \n",
       "10         []        []       []     []     []   \n",
       "11         []        []       []     []     []   \n",
       "12         []        []       []     []     []   \n",
       "13         []        []       []     []     []   \n",
       "14         []        []       []     []     []   \n",
       "15         []        []       []     []     []   \n",
       "16         []        []       []     []     []   \n",
       "17         []        []       []     []     []   \n",
       "18         []        []       []     []     []   \n",
       "19         []        []       []     []     []   \n",
       "20         []        []       []     []     []   \n",
       "21         []        []       []     []     []   \n",
       "22         []        []       []     []     []   \n",
       "23         []        []       []     []     []   \n",
       "24         []        []       []     []     []   \n",
       "25         []        []       []     []     []   \n",
       "26         []        []       []     []     []   \n",
       "27         []        []       []     []     []   \n",
       "28         []        []       []     []     []   \n",
       "29         []        []       []     []     []   \n",
       "...       ...       ...      ...    ...    ...   \n",
       "1014       []        []       []     []     []   \n",
       "1015       []        []       []     []     []   \n",
       "1016       []        []       []     []     []   \n",
       "1017       []        []       []     []     []   \n",
       "1018       []        []       []     []     []   \n",
       "1019       []        []       []     []     []   \n",
       "1020       []        []       []     []     []   \n",
       "1021       []        []       []     []     []   \n",
       "1022       []        []       []     []     []   \n",
       "1023       []        []       []     []     []   \n",
       "1024       []        []       []     []     []   \n",
       "1025       []        []       []     []     []   \n",
       "1026       []        []       []     []     []   \n",
       "1027       []        []       []     []     []   \n",
       "1028       []        []       []     []     []   \n",
       "1029       []        []       []     []     []   \n",
       "1030       []        []       []     []     []   \n",
       "1031       []        []       []     []     []   \n",
       "1032       []        []       []     []     []   \n",
       "1033       []        []       []     []     []   \n",
       "1034       []        []       []     []     []   \n",
       "1035       []        []       []     []     []   \n",
       "1036       []        []       []     []     []   \n",
       "1037       []        []       []     []     []   \n",
       "1038       []        []       []     []     []   \n",
       "1039       []        []       []     []     []   \n",
       "1040       []        []       []     []     []   \n",
       "1041       []        []       []     []     []   \n",
       "1042       []        []       []     []     []   \n",
       "1043       []        []       []     []     []   \n",
       "\n",
       "                                                subject  \\\n",
       "0                           [Physics - General Physics]   \n",
       "1     [Mathematics - Combinatorics, Mathematics - Pr...   \n",
       "2     [Physics - General Physics, Physics - Classica...   \n",
       "3     [Mathematics - Combinatorics, Computer Science...   \n",
       "4             [Computer Science - Discrete Mathematics]   \n",
       "5     [Mathematics - Combinatorics, Computer Science...   \n",
       "6     [High Energy Physics - Theory, General Relativ...   \n",
       "7                     [Mathematics - Statistics Theory]   \n",
       "8     [Mathematics - Representation Theory, Mathemat...   \n",
       "9     [High Energy Physics - Theory, Condensed Matte...   \n",
       "10    [High Energy Physics - Theory, General Relativ...   \n",
       "11    [General Relativity and Quantum Cosmology, Hig...   \n",
       "12    [High Energy Physics - Phenomenology, General ...   \n",
       "13    [Mathematics - Algebraic Topology, Mathematics...   \n",
       "14    [Mathematics - Combinatorics, Computer Science...   \n",
       "15    [Mathematics - Differential Geometry, Mathemat...   \n",
       "16                       [High Energy Physics - Theory]   \n",
       "17    [General Relativity and Quantum Cosmology, Hig...   \n",
       "18                               [Mathematical Physics]   \n",
       "19    [Mathematics - Combinatorics, Computer Science...   \n",
       "20    [Computer Science - Learning, Statistics - Mac...   \n",
       "21                       [High Energy Physics - Theory]   \n",
       "22    [Mathematics - Functional Analysis, Mathematic...   \n",
       "23    [Physics - Physics and Society, Computer Scien...   \n",
       "24    [Mathematics - Classical Analysis and ODEs, 26...   \n",
       "25    [Astrophysics - High Energy Astrophysical Phen...   \n",
       "26    [Mathematics - Representation Theory, 20G05, 2...   \n",
       "27    [Computer Science - Discrete Mathematics, Math...   \n",
       "28    [Mathematics - Differential Geometry, Mathemat...   \n",
       "29    [Mathematics - Commutative Algebra, Mathematic...   \n",
       "...                                                 ...   \n",
       "1014  [Computer Science - Computer Vision and Patter...   \n",
       "1015  [Computer Science - Computer Vision and Patter...   \n",
       "1016  [Mathematics - Category Theory, Mathematics - ...   \n",
       "1017  [Mathematics - Algebraic Geometry, Mathematics...   \n",
       "1018   [Quantum Physics, High Energy Physics - Lattice]   \n",
       "1019                      [Computer Science - Robotics]   \n",
       "1020  [Condensed Matter - Mesoscale and Nanoscale Ph...   \n",
       "1021  [Mathematics - Category Theory, Computer Scien...   \n",
       "1022  [Computer Science - Learning, Mathematics - Op...   \n",
       "1023  [Physics - Applied Physics, Condensed Matter -...   \n",
       "1024  [Astrophysics - Cosmology and Nongalactic Astr...   \n",
       "1025                                  [Quantum Physics]   \n",
       "1026                                  [Quantum Physics]   \n",
       "1027  [Condensed Matter - Strongly Correlated Electr...   \n",
       "1028  [Computer Science - Learning, Statistics - Mac...   \n",
       "1029                         [Statistics - Methodology]   \n",
       "1030  [Computer Science - Computer Vision and Patter...   \n",
       "1031                                  [Quantum Physics]   \n",
       "1032  [Mathematics - Geometric Topology, Mathematica...   \n",
       "1033  [Computer Science - Learning, Computer Science...   \n",
       "1034  [Mathematics - Functional Analysis, Mathematic...   \n",
       "1035                                 [Physics - Optics]   \n",
       "1036  [Mathematics - Geometric Topology, Mathematics...   \n",
       "1037  [Condensed Matter - Statistical Mechanics, Con...   \n",
       "1038  [High Energy Physics - Theory, Astrophysics - ...   \n",
       "1039  [Condensed Matter - Strongly Correlated Electr...   \n",
       "1040  [High Energy Physics - Theory, General Relativ...   \n",
       "1041                     [High Energy Physics - Theory]   \n",
       "1042  [High Energy Physics - Theory, General Relativ...   \n",
       "1043  [Mathematics - Algebraic Geometry, Mathematics...   \n",
       "\n",
       "                                                  title    type  \n",
       "0     [A Procedure to Solve the Eigen Solution to Di...  [text]  \n",
       "1         [Algorithmic barriers from phase transitions]  [text]  \n",
       "2     [Explanation on Overdetermination of Maxwell's...  [text]  \n",
       "3     [On belief propagation guided decimation for r...  [text]  \n",
       "4                [On independent sets in random graphs]  [text]  \n",
       "5              [The decimation process in random k-SAT]  [text]  \n",
       "6     [Status of Horava gravity: A personal perspect...  [text]  \n",
       "7     [Adaptive estimation in the nonparametric rand...  [text]  \n",
       "8     [Unitary representations of cyclotomic rationa...  [text]  \n",
       "9     [Thermodynamics of Classical Systems on Noncom...  [text]  \n",
       "10                       [Conservative entropic forces]  [text]  \n",
       "11         [Entropy bounds in terms of the w parameter]  [text]  \n",
       "12    [Lorentz violating kinematics: Threshold theor...  [text]  \n",
       "13    [Rational Blanchfield forms, S-equivalence, an...  [text]  \n",
       "14                    [Going after the k-SAT Threshold]  [text]  \n",
       "15             [Bott-Chern cohomology of solvmanifolds]  [text]  \n",
       "16    [Gilkey-de Witt heat kernel expansion and zero...  [text]  \n",
       "17    [Clausius entropy for arbitrary bifurcate null...  [text]  \n",
       "18    [Classification of Darboux transformations for...  [text]  \n",
       "19    [Upper-bounding the k-colorability threshold b...  [text]  \n",
       "20    [Stochastic approximation for speeding up LSTD...  [text]  \n",
       "21    [Quantum-induced interactions in the moduli sp...  [text]  \n",
       "22                     [On convoluters on $L^p$-spaces]  [text]  \n",
       "23             [The dynamic pattern of human attention]  [text]  \n",
       "24                [A new proof of Bronshtein's theorem]  [text]  \n",
       "25    [AMS-02 data confronts acceleration of cosmic ...  [text]  \n",
       "26    [Hausdorffness for Lie algebra homology of Sch...  [text]  \n",
       "27    [The condensation phase transition in random g...  [text]  \n",
       "28    [Lifting differentiable curves from orbit spaces]  [text]  \n",
       "29    [Degree of reductivity of a modular representa...  [text]  \n",
       "...                                                 ...     ...  \n",
       "1014  [DFUNet: Convolutional Neural Networks for Dia...  [text]  \n",
       "1015  [Multi-class Semantic Segmentation of Skin Les...  [text]  \n",
       "1016  [A relative monotone-light factorisation syste...  [text]  \n",
       "1017         [A geometric version of the circle method]  [text]  \n",
       "1018  [The Kibble Zurek Mechanism of Topological Def...  [text]  \n",
       "1019  [Deep Predictive Models for Collision Risk Ass...  [text]  \n",
       "1020  [Femtosecond valley polarization and topologic...  [text]  \n",
       "1021  [Backprop as Functor: A compositional perspect...  [text]  \n",
       "1022  [Accelerated Gradient Descent Escapes Saddle P...  [text]  \n",
       "1023  [A salt water battery with high stability and ...  [text]  \n",
       "1024  [Seven Hints for Primordial Black Hole Dark Ma...  [text]  \n",
       "1025  [Distributed Quantum Sensing Using Continuous-...  [text]  \n",
       "1026  [Improved Techniques for Preparing Eigenstates...  [text]  \n",
       "1027  [Spin eigen-excitations of the magnetic skyrmi...  [text]  \n",
       "1028  [Plan, Attend, Generate: Planning for Sequence...  [text]  \n",
       "1029  [The joint projected normal and skew-normal: a...  [text]  \n",
       "1030  [Openmv: A Python powered, extensible machine ...  [text]  \n",
       "1031  [Resource theory of contextuality for arbitrar...  [text]  \n",
       "1032  [Analytic sphere eversion with minimum of topo...  [text]  \n",
       "1033  [Implicit Regularization in Nonconvex Statisti...  [text]  \n",
       "1034                      [Total-positivity preservers]  [text]  \n",
       "1035  [Frequency comb generation in a continuously p...  [text]  \n",
       "1036                           [Models of Random Knots]  [text]  \n",
       "1037  [Complexity and geometry of quantum state mani...  [text]  \n",
       "1038  [Massive gravity and the suppression of anisot...  [text]  \n",
       "1039  [Chiral tricritical point: a new universality ...  [text]  \n",
       "1040  [Asymptotically-flat supergravity solutions de...  [text]  \n",
       "1041  [Non-commutative relativistic spacetimes and w...  [text]  \n",
       "1042  [The Characteristic Functions and Their Typica...  [text]  \n",
       "1043  [Foundations for almost ring theory -- Release 7]  [text]  \n",
       "\n",
       "[1044 rows x 16 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_read = pd.read_json(r\"metadata_arxiv_0511.txt\")\n",
    "df_read"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### download updates only using by date\n",
    "note that arxiv only accepts from dates and will download all papers to present"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total records: 52261\n"
     ]
    }
   ],
   "source": [
    "# adjust start date in case request fails/we get kicked out\n",
    "\n",
    "from_ = \"2017-09-13\"\n",
    "from_datetime = datetime.datetime.strptime(from_, \"%Y-%m-%d\")\n",
    "\n",
    "metadata_all = []\n",
    "\n",
    "i = 0 \n",
    "for record in client.listRecords(metadataPrefix='oai_dc',from_=from_datetime):\n",
    "    \n",
    "    # get current record id/metadata\n",
    "    header = record[0]\n",
    "    id = header.identifier().split(\"oai:arXiv.org:\")[1]\n",
    "        \n",
    "    metadata_raw = record[1]\n",
    "    \n",
    "    if metadata_raw:\n",
    "        metadata = metadata_raw.getMap()\n",
    "    else: \n",
    "        metadata = []\n",
    "            \n",
    "    metadata['id'] = id\n",
    "        \n",
    "    # append record data\n",
    "    metadata_all.append(metadata)\n",
    "            \n",
    "    \n",
    "    i += 1\n",
    "\n",
    "print(\"total records:\", len(metadata_all))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save update\n",
    "j = 522 # set to last file + 1\n",
    "with open(r\"metadata_arxiv_%04d.txt\" % j, 'w') as outfile:     \n",
    "            json.dump(metadata_all, outfile)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43005"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# consistentcy check\n",
    "ids = [metadata['id'] for metadata in metadata_all]\n",
    "ids.index('1711.02023')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}