Validate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import database\n",
    "import importlib\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "from snowballing.operations import load_work, load_citations, reload\n",
    "from snowballing.operations import work_by_varname, load_work_map_all_years\n",
    "from snowballing.models import Place, DB\n",
    "from snowballing.jupyter_utils import idisplay, work_button, new_button\n",
    "from snowballing.selenium_scholar import SeleniumScholarQuerier\n",
    "from snowballing.dbmanager import set_attribute\n",
    "\n",
    "def pages(work):\n",
    "    pp = getattr(work, 'pp', '')\n",
    "    if not pp:\n",
    "        return 0\n",
    "    split = list(map(int, pp.split('--')))\n",
    "    if len(split) == 1:\n",
    "        return split[0]\n",
    "    return split[-1] - split[0] + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Place"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload()\n",
    "result = [\n",
    "    idisplay(work_button(w))\n",
    "    for _, w in load_work_map_all_years()\n",
    "    if hasattr(w, \"place1\")\n",
    "    or not isinstance(w.place, Place)\n",
    "]\n",
    "len(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[p for p in DB.places() if not hasattr(p, \"type\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(PRBook, 'Peer Reviewed Book'), (PRBook, 'Peer Reviewed Book')]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(p, p.type) for p in DB.places()\n",
    " if p.type not in {\n",
    "     \"Conference\", \n",
    "     \"Journal\", \n",
    "     \"Magazine\", \n",
    "     \"Sponsor\",\n",
    "     \"Proceedings\",\n",
    "     \"Archive\",\n",
    "     \"Tech Report\",\n",
    "     \"Unpublished\",\n",
    "     \"Thesis\",\n",
    "     \"Patent\",\n",
    "     \"Lang\",\n",
    "     \"Web\",\n",
    "     \"Standard\",\n",
    "     \"Book\",\n",
    "     \"Other\",\n",
    " }]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check IPAW papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload()\n",
    "result = [\n",
    "    idisplay(work_button(w))\n",
    "    for _, w in load_work_map_all_years()\n",
    "    if (w.category == \"fullpaper\" and w.place.acronym != \"IPAW\")\n",
    "    or (w.category == \"regular\" and w.place.acronym != \"IPAW\")\n",
    "    or (w.category == \"short\" and w.place.acronym != \"IPAW\")\n",
    "    or (w.category == \"demo\" and w.place.acronym != \"IPAW\")\n",
    "    or (w.category == \"poster\" and w.place.acronym != \"IPAW\")\n",
    "    or (w.category == \"keynote\" and w.place.acronym != \"IPAW\")\n",
    "    or (w.category == \"latebreaking\" and w.place.acronym != \"IPAW\")\n",
    "    or (w.place.acronym == \"IPAW\" and w.category not in {\n",
    "        \"fullpaper\", \"regular\", \"short\", \"demo\", \"poster\", \"keynote\", \"latebreaking\"\n",
    "    })\n",
    "]\n",
    "len(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload()\n",
    "result = [\n",
    "    idisplay(work_button(w))\n",
    "    for _, w in load_work_map_all_years()\n",
    "    if not hasattr(w, \"pp\")\n",
    "    if w.category in {\n",
    "        \"fullpaper\", \"regular\", \"short\", \"demo\", \"poster\", \"latebreaking\"\n",
    "    }\n",
    "    \n",
    "]\n",
    "len(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reload()\n",
    "result = [\n",
    "    idisplay(work_button(w))\n",
    "    for _, w in load_work_map_all_years()\n",
    "    if not hasattr(w, \"name\")\n",
    "    or w.name.endswith(\".\")\n",
    "]\n",
    "len(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reload()\n",
    "result = [\n",
    "    idisplay(work_button(w))\n",
    "    for _, w in load_work_map_all_years()\n",
    "    if not hasattr(w, \"authors\")\n",
    "    or w.authors.endswith(\".\")\n",
    "    and not w.authors.endswith(\" al.\")\n",
    "    and not w.authors[-3] in \"- \" and w.authors[-2].isalpha()\n",
    "]\n",
    "len(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scholar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "querier = None\n",
    "reload()\n",
    "worklist = sorted(\n",
    "    [k for k, w in load_work_map_all_years() if not getattr(w, \"scholar_ok\", False)],\n",
    "    key=lambda x: (int(x[-5:-1]), x)\n",
    ")\n",
    "len(worklist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if worklist and querier is None:\n",
    "    querier = SeleniumScholarQuerier()\n",
    "    querier.apply_settings(10, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from snowballing.snowballing import ScholarUpdate\n",
    "supdate = ScholarUpdate(querier, worklist, force=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "supdate.browser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Temp\n",
    "set_attribute('wu2016a', 'cluster_id', '14505887097699069314')\n",
    "set_attribute('wu2016a', 'scholar', 'http://scholar.google.com/scholar?cites=14505887097699069314&as_sdt=2005&sciodt=0,5&hl=en')\n",
    "set_attribute('wu2016a', 'scholar_ok', True)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "{w.scholar_ok for k, w in load_work_map_all_years() if getattr(w, \"scholar_ok\", False)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forward\n",
    "\n",
    "Finding scholar cluster: http://webapps.stackexchange.com/questions/45333/how-to-create-a-citation-alert-for-a-paper-without-citation-in-google-scholar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def set_snowball_date_button(key, date):\n",
    "    def click(w):\n",
    "        set_attribute(key, \"snowball\", \"datetime({0.year}, {0.month}, {0.day})\".format(date))\n",
    "    return new_button(\"Set date\", click)\n",
    "    \n",
    "date = datetime(2018, 5, 17)\n",
    "reload()\n",
    "result = [\n",
    "    idisplay(\n",
    "        work_button(w),\n",
    "        set_snowball_date_button(k, date),\n",
    "        '\"{}\", citation_file=\"{}\"'.format(k, getattr(w, 'citation_file', w.metakey)) if hasattr(w, 'scholar') else w.name,\n",
    "        label=False\n",
    "    )\n",
    "    for k, w in load_work_map_all_years()\n",
    "    if w.place.acronym == \"IPAW\"\n",
    "    if pages(w) >= 6\n",
    "    #if w.category in {\"fullpaper\", \"regular\"}\n",
    "    if not hasattr(w, 'snowball') or w.snowball != date\n",
    "    if w.year >= 2008\n",
    "]\n",
    "len(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Dash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def check_dash(ele):\n",
    "    k, w = ele\n",
    "    def create_button(attr):\n",
    "        def click(_):\n",
    "            set_attribute(k, attr, str(getattr(w, attr)).replace(\"-\", \"--\"))\n",
    "        return new_button(\"Set \" + attr, click)\n",
    "    reasons = []\n",
    "    if w.place.name in (\"Thesis\", \"Tech Report\"):\n",
    "        return k, w, reasons\n",
    "    if isinstance(getattr(w, 'pp', ''), int):\n",
    "        reasons.append(create_button('pp'))\n",
    "    elif getattr(w, 'pp', '').count('-') not in (0, 2):\n",
    "        reasons.append(create_button('pp'))\n",
    "    if isinstance(getattr(w, 'volume', ''), int):\n",
    "        reasons.append(create_button('volume'))\n",
    "    elif getattr(w, 'volume', '').count('-') not in (0, 2):\n",
    "        reasons.append(create_button('volume'))\n",
    "    if isinstance(getattr(w, 'number', ''), int):\n",
    "        reasons.append(create_button('number'))\n",
    "    elif getattr(w, 'number', '').count('-') not in (0, 2):\n",
    "        reasons.append(create_button('number'))\n",
    "    return k, w, reasons\n",
    "\n",
    "reload()\n",
    "result = [\n",
    "    idisplay(work_button(w), *r)\n",
    "    for k, w, r in map(check_dash, load_work_map_all_years())\n",
    "    if r\n",
    "]\n",
    "len(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "toc": {
   "nav_menu": {
    "height": "142px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "widgets": {
   "state": {
    "163969c3dd3449fa84df6bb2a9eddb50": {
     "views": [
      {
       "cell_index": 14
      }
     ]
    },
    "e148968a58354579a5d15c603ad4f1b6": {
     "views": [
      {
       "cell_index": 14
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}