compile/AeG-expression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine gene-anatomy expression resources (Bgee and TISSUES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import gzip\n",
    "\n",
    "import pandas\n",
    "\n",
    "# local imports\n",
    "sys.path.insert(0, '../')\n",
    "import utils\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read TISSUES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "commit = 'd6b0c99352db27469f2c3399cecb6f9fae2db547'\n",
    "url = utils.rawgit('dhimmel', 'tissues', commit, 'data/merged.tsv.gz')\n",
    "tissues_df = pandas.read_table(url, compression='gzip')\n",
    "tissues_df = tissues_df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEACAYAAACtVTGuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG4tJREFUeJzt3X+MXeV95/H3B1uGJgHbbBZPZQeGKjiBbKLBCEcrKmVK\nl1/7A+hKAbdd4WnYaotBgVRaYSdSvNm0SkBq1tmtIN0N28EorGGRWsiu1xiER6u0YAxhCg0uuLuy\nwQZPCDZOUbrINt/94z7j53p87517PWfuOffez0u64pzn/Hrmy+V873m+5xwUEZiZmTVzRtkdMDOz\nanOiMDOzlpwozMysJScKMzNryYnCzMxacqIwM7OWZk0Uks6UtFPSS5JekbQxtS+VtF3Sa5KelLS4\nbpsNkvZI2i3p6rr2VZJelvS6pE117YskbUnbPCvp/Lpla9P6r0m6pbg/3czM2jFrooiID4Bfi4hL\ngRHgOkmrgfXA0xHxKeAZYAOApEuAm4CLgeuA+yQp7e5+4NaIWAmslHRNar8VOBQRFwGbgHvTvpYC\nXwcuBz4PbKxPSGZmNv/aGnqKiF+kyTOBhUAANwAPpvYHgRvT9PXAlog4FhF7gT3AaklDwNkRsSut\nt7lum/p9PQZcmaavAbZHxJGIeA/YDlzb0V9oZmZz0laikHSGpJeAg8BT6WS/LCKmACLiIHBeWn05\n8Gbd5gdS23Jgf137/tR20jYRcRw4IuncFvsyM7MuafeK4sM09LSC2tXBZ6hdVZy0WoH90uyrmJlZ\nNyzsZOWI+LmkCWrDP1OSlkXEVBpW+mla7QDwibrNVqS2Zu3127wlaQFwTkQcknQAGJ2xzY6Z/ZLk\nF1aZmZ2GiJj1h3k7dz19fLqALOmXgKuA3cATwFhabS3weJp+AliT7mS6EPgk8HwanjoiaXUqbt8y\nY5u1afqL1IrjAE8CV0lanArbV6W2Rn+sPxFs3Lix9D5U5eNYOBaORetPu9q5ovhl4EFJZ1BLLI9E\nxFZJzwGPSvoSsI/anU5ExKuSHgVeBY4C6yL36HZgHDgL2BoR21L7A8BDkvYA7wJr0r4OS/om8AK1\noa1vRK2obU3s3bu37C5UhmORORaZY9G5WRNFRLwCrGrQfgj4J022+RbwrQbtLwKfbdD+ASnRNFg2\nTi25mJlZCfxkdp8ZGxsruwuV4VhkjkXmWHROnYxTVZWk6Ie/w8ysmyQRRRSzrbdMTEyU3YXKcCwy\nxyJzLDrnRGFmZi156MnMbEB56MnMzArhRNFnPP6aORaZY5E5Fp1zojAzs5ZcozAzG1CuUZiZWSGc\nKPqMx18zxyJzLDLHonNOFGZm1pJrFGZmA8o1CjMzK4QTRZ/x+GvmWGSOReZYdM6JwszMWnKNwsxs\nQLlGYWZmhXCi6DMef80ci8yxyByLzjlRmJlZS65RmJkNKNcozMysEE4Ufcbjr5ljkTkWmWPROScK\nMzNryTUKM7MB5RqFmZkVwomiz3j8NXMsMscicyw650RhZmYtzVqjkLQC2AwsAz4E/nNE/CdJG4Hf\nBX6aVv1qRGxL22wAvgQcA+6MiO2pfRUwDpwFbI2Iu1L7onSMy4CfATdHxBtp2Vrga0AAfxgRmxv0\n0TUKM7MOtVujaCdRDAFDETEp6WPAi8ANwM3A30XEd2asfzHwMHA5sAJ4GrgoIkLSTuCOiNglaSvw\n3Yh4UtJtwGcjYp2km4HfiIg1kpYCLwCrAKVjr4qIIzOO6URhZtahworZEXEwIibT9PvAbmD59HEa\nbHIDsCUijkXEXmAPsDolnLMjYldabzNwY902D6bpx4Ar0/Q1wPaIOBIR7wHbgWtn6/Mg8/hr5lhk\njkXmWHSuoxqFpGFgBNiZmu6QNCnp+5IWp7blwJt1mx1IbcuB/XXt+8kJ58Q2EXEcOCLp3Bb7qqSh\noWEkNfwMDQ2X3T0zs9PSdqJIw06PUas5vA/cB/xKRIwAB4E/KrBfs14KVdHU1D5qpZRTP7Vl8290\ndLQrx+kFjkXmWGSORecWtrOSpIXUksRDEfE4QES8U7fKfwF+mKYPAJ+oW7YitTVrr9/mLUkLgHMi\n4pCkA8DojG12NOrj2NgYw8PDACxZsoSRkZETX4jpS835ns+m50dnzNPV/nje8573fP38xMQE4+Pj\nACfOl22JiFk/1OoJ35nRNlQ3/RXg4TR9CfASsAi4EPhbctH8OWA1tSuGrcC1qX0dcF+aXkOtxgGw\nFPg/wOK66SUN+hdVAAREk093+rhjx46uHKcXOBaZY5E5Flk6L82aA2a9opB0BfDbwCuSXqqdDPkq\n8FuSRqjdMrsX+DfpjP2qpEeBV4GjwLrUIYDbOfn22G2p/QHgIUl7gHdTsiAiDkv6JrU7nwL4RtSK\n2mZm1iV+11Ox/aCWzxoupQp9NDOb5nc9mZlZIZwo+sx04coci3qOReZYdM6JwszMWnKNoth+4BqF\nmfUK1yjMzKwQThR9xuOvmWORORaZY9E5JwozM2vJNYpi+4FrFGbWK1yjMDOzQjhR9BmPv2aOReZY\nZI5F55wozMysJdcoiu0HrlGYWa9wjcLMzArhRNFnPP6aORaZY5E5Fp1zojAzs5Zcoyi2H7hGYWa9\nwjUKMzMrhBNFn/H4a+ZYZI5F5lh0zonCzMxaco2i2H7gGoWZ9QrXKMzMrBBOFH3G46+ZY5E5Fplj\n0TknCjMza8k1imL7gWsUZtYrXKMwM7NCOFH0GY+/Zo5F5lhkjkXnnCjMzKwl1yiK7QeuUZhZr3CN\nwszMCjFropC0QtIzkn4i6RVJX07tSyVtl/SapCclLa7bZoOkPZJ2S7q6rn2VpJclvS5pU137Iklb\n0jbPSjq/btnatP5rkm4p7k/vTx5/zRyLzLHIHIvOtXNFcQz4/Yj4DPCPgdslfRpYDzwdEZ8CngE2\nAEi6BLgJuBi4DrhPtTEZgPuBWyNiJbBS0jWp/VbgUERcBGwC7k37Wgp8Hbgc+DywsT4hmZnZ/Ou4\nRiHpz4E/Tp8vRMSUpCFgIiI+LWk9EBFxT1r/fwH/DtgHPBMRl6T2NWn72yRtAzZGxE5JC4C3I+K8\n+nXSNven4zwyo0+uUZiZdWheahSShoER4DlgWURMAUTEQeC8tNpy4M26zQ6ktuXA/rr2/antpG0i\n4jhwRNK5LfZlZmZdsrDdFSV9DHgMuDMi3pc08+dxkT+XZ81wM42NjTE8PAzAkiVLGBkZYXR0FMhj\nkvM9n03Pj86YZ977U9+Xbv/9VZufbqtKf8qcn5yc5K677qpMf8qc37RpUynnhyrMT0xMMD4+DnDi\nfNmWiJj1Qy2hbKOWJKbbdlO7qgAYAnan6fXA3XXrbaNWXzixTmpfA9xfv06aXgD8tG6d79Vt8z3g\n5gb9iyoAAqLJpzt93LFjR1eO0wsci8yxyByLLJ2XZs0BbdUoJG0GfhYRv1/Xdg+1AvQ9ku4GlkbE\n+lTM/kFKDsuBp4CLIiIkPQd8GdgF/E/gP0bENknrgH8UEetSXeLGiFiTitkvAKuoDZO9AFwWEe/N\n6F+083fMN9cozKyXtFujmDVRSLoC+N/AK9TOggF8FXgeeBT4BLVC9U3TJ3BJG6jdyXSU2lXI9tR+\nGTAOnAVsjYg7U/uZwEPApcC7wJqI2JuWjQFfS8f9g4jY3KCPThRmZh0qLFH0AieKbGJi4sTY5KBz\nLDLHInMsMj+ZbWZmhfAVRbH9oOwrCjOzdvmKwk4xNDSMpIafoaHhsrtnZhXlRNFn6p8hmGlqah/5\nfoSTP7Vl/aVVLAaNY5E5Fp1zojAzs5Zcoyi2H1S5RlH1/plZd7lGYWZmhXCi6DMef80ci8yxyByL\nzjlRmJlZS65RFNsPqlwDqHr/zKy7XKMwM7NCOFH0GY+/Zo5F5lhkjkXnnCjMzKwl1yiK7QdVrgFU\nvX9m1l2uUZiZWSGcKPqMx18zxyJzLDLHonNOFGZm1pJrFMX2gyrXAKrePzPrLtcozMysEE4Ufcbj\nr5ljkTkWmWPROScKMzNryTWKYvtBlWsAVe+fmXWXaxRmZlYIJ4o+4/HXzLHIHIvMseicE4WZmbXk\nGkWx/aDKNYCq98/Muss1CjMzK4QTRZ/x+GvmWGSOReZYdG7WRCHpAUlTkl6ua9soab+kH6fPtXXL\nNkjaI2m3pKvr2ldJelnS65I21bUvkrQlbfOspPPrlq1N678m6ZZi/mQzM+vErDUKSb8KvA9sjojP\npbaNwN9FxHdmrHsx8DBwObACeBq4KCJC0k7gjojYJWkr8N2IeFLSbcBnI2KdpJuB34iINZKWAi8A\nqwABLwKrIuJIgz66RtGGqvfPzLqrsBpFRPwIONzoGA3abgC2RMSxiNgL7AFWSxoCzo6IXWm9zcCN\ndds8mKYfA65M09cA2yPiSES8B2wHTly5mJlZd8ylRnGHpElJ35e0OLUtB96sW+dAalsO7K9r35/a\nTtomIo4DRySd22Jf1oLHXzPHInMsMseicwtPc7v7gH+fhpT+APgj4F8X1KdZL4MaGRsbY3h4GIAl\nS5YwMjLC6OgokL8Y8z2fTc+Pzpinq/3ptf7N199blf6UOT85OVmp/pQ5Pzk5Wan+dHN+YmKC8fFx\ngBPny3a09RyFpAuAH07XKJotk7QeiIi4Jy3bBmwE9gE7IuLi1L4G+EJE3Da9TkTslLQAeDsizkvr\njEbE76Vtvpf28UiDPrhG0Yaq98/Muqvo5yhE3S/9VHOY9i+Bv07TTwBr0p1MFwKfBJ6PiIPUhpRW\nq3a2ugV4vG6btWn6i8AzafpJ4CpJi1Nh+6rUZmZmXdTO7bEPA38JrJT0hqTfAe5Nt7pOAl8AvgIQ\nEa8CjwKvAluBdXU/9W8HHgBeB/ZExLbU/gDwcUl7gLuA9Wlfh4FvUrvzaSfwjVTUthZOHWYaXI5F\n5lhkjkXnZq1RRMRvNWj+0xbrfwv4VoP2F4HPNmj/ALipyb7GgfHZ+mhmZvPH73oqth9UuQZQ9f6Z\nWXf5XU9mZlYIJ4o+4/HXzLHIHIvMseicE4WZmbXkGkWx/aDKNYCq98/Muss1CjMzK4QTRZ/x+Gvm\nWGSOReZYdM6JwszMWnKNoth+UOUaQNX7Z2bd5RpF5ZyJpFM+Q0PDZXfMzKwlJ4qu+YDar/mTP1NT\n+wo9isdfM8cicywyx6JzThRmZtbSQNcohoaGm/6iX7bsAg4e3NtpP2hVA2i8rHu1AdcozKxeuzWK\ngU4URZ84nSjMrJe4mD2gPP6aORaZY5E5Fp1zojAzs5Y89OShp673w8yqwUNPZmZWCCeKPuPx18yx\nyByLzLHonBNF6Ro/se2nts2sKlyjqECNolt1A9cozKyeaxRmZlYIJ4o+4/HXzLHIHIvMseicE4WZ\nmbXkGkWlaxRnUXvr7Knm411U/fBdMLP2+V1P7W1HtRNFd/vXD98FM2ufi9kDyuOvmWORORaZY9E5\nJwozM2tp1qEnSQ8A/xyYiojPpbalwCPABcBe4KaIOJKWbQC+BBwD7oyI7al9FTBObeB9a0TcldoX\nAZuBy4CfATdHxBtp2Vrga9TGS/4wIjY36aOHngron4eezAZLkUNPfwpcM6NtPfB0RHwKeAbYkA56\nCXATcDFwHXCfamcngPuBWyNiJbBS0vQ+bwUORcRFwCbg3rSvpcDXgcuBzwMbJS1uo79mZlagWRNF\nRPwIODyj+QbgwTT9IHBjmr4e2BIRxyJiL7AHWC1pCDg7Inal9TbXbVO/r8eAK9P0NcD2iDgSEe8B\n24FrO/jbBpLHXzPHInMsMseicwtPc7vzImIKICIOSjovtS8Hnq1b70BqOwbsr2vfn9qnt3kz7eu4\npCOSzq1vn7EvA6bfETXT0qXLOHToYAn9MbN+dbqJYqYiB7dnHS9rZGxsjOHhYQCWLFnCyMgIo6Oj\nQP4FMXM+m54fnTFPy+1Pd3/Nl3ey/gfksOflhw+ra3+v53trflpV+lPW/HRbVfrTzfmJiQnGx8cB\nTpwv29HWcxSSLgB+WFfM3g2MRsRUGlbaEREXS1oPRETck9bbBmwE9k2vk9rXAF+IiNum14mInZIW\nAG9HxHlpndGI+L20zffSPh5p0L+BLGZ3+j9CcjHbzOoV/RyFOPmX/hPAWJpeCzxe175G0iJJFwKf\nBJ6PiIPAEUmrU3H7lhnbrE3TX6RWHAd4ErhK0uJU2L4qtZm15dQrqcHlWGSORedmHXqS9DC1MYp/\nIOkNalcI3wb+u6QvUbtauAkgIl6V9CjwKnAUWFf3U/92Tr49dltqfwB4SNIe4F1gTdrXYUnfBF6g\n9jP4G6mobWZmXeRXeHjoadbtzKw/tTv0VFQx2yqj8d1QZmany6/w6DvTd0M1+gwWj0VnjkXmWHTO\nicLMzFpyjaIPaxTdeneUmfU2v2bczMwK4URhfctj0ZljkTkWnXOiMDOzllyjcI3ixLJ++C6YWftc\no7AO1Z6/mPkZGhouu2NmVjInCksaP38xNbWv1F7NhceiM8cicyw650RhZmYtuUbhGsWs++uH74iZ\nnco1CjMzK4QThfUtj0VnjkXmWHTOicLMzFpyjcI1iln31w/fETM7lWsUVpDGz1f4GQuzweFEYbNo\n/v+3qPozFh6LzhyLzLHonBOFmZm15BqFaxRz2l8/fH/MBpVrFGZmVggnCutbHovOHIvMseicE4WZ\nmbXkGoVrFHPY31nU7oo61bJlF3Dw4N4m25lZFbRbo3CicKKYt/31w3fLrJ+5mG0Dz2PRmWORORad\n6/tEMTQ03PTJYjMzm13fDz2d3nBQbZmHnua2v374bpn1s64MPUnaK+mvJL0k6fnUtlTSdkmvSXpS\n0uK69TdI2iNpt6Sr69pXSXpZ0uuSNtW1L5K0JW3zrKTz59JfMzPr3FyHnj4ERiPi0ohYndrWA09H\nxKeAZ4ANAJIuAW4CLgauA+5THv+5H7g1IlYCKyVdk9pvBQ5FxEXAJuDeOfbXBojHojPHInMsOjfX\nRKEG+7gBeDBNPwjcmKavB7ZExLGI2AvsAVZLGgLOjohdab3NddvU7+sx4Nfn2N85c82jXX7rrFm/\nWDjH7QN4StJx4E8i4vvAsoiYAoiIg5LOS+suB56t2/ZAajsG7K9r35/ap7d5M+3ruKT3JJ0bEYfm\n2O82nNni5N9qzN5qpt86e6qpqe7EaXR0tCvH6QWOReZYdG6uieKKiHhb0j8Etkt6jVPPDkVWNLt4\nJm52onMyMLPBMqdEERFvp3++I+nPgdXAlKRlETGVhpV+mlY/AHyibvMVqa1Ze/02b0laAJzT7Gpi\nbGyM4eFhAJYsWcLIyEjdL4eJ9M+Z88xxufd3estr48TT/36mx4yLnp9um6/999L85OQkd911V2X6\nU+b8pk2bTjo/lN2fbs5PTEwwPj4OcOJ82ZaIOK0P8BHgY2n6o8BfAFcD9wB3p/a7gW+n6UuAl4BF\nwIXA35Jvz32OWpIRsBW4NrWvA+5L02uo1Tga9SWaAQKiyed0lnl/RSzrhh07dnTlOL3Ascgciyz9\ntzjr+f60n6OQdCHwZ7UTAguBH0TEtyWdCzxK7UpgH3BTRLyXttlA7U6mo8CdEbE9tV8GjFN7edDW\niLgztZ8JPARcCrwLrIlaIXxmX6LZ3zGX5yiq/pxCb+4Pmr0jyu+HMusuv+spL6PaJ85B21/rY/XD\n99GsV/hdTzbwfL985lhkjkXnnCjMzKwlDz2VPhQzaPtrfax++D6a9QoPPZmZWSGcKKxCin3th8ei\nM8ciO91YNHt9zyC8kmauT2abFaj8136YNTM1tY9G389B+G66RlH6mP2g7e/0j9UP31XrXc3PJb37\n3XSNwszMCuFEYT2i8/qFx+UzxyIrPhb9/0p91yisR7h+YVXV/99N1yhKH7MftP3Nx7H87iibf61q\nFL1aW2u3RuErCusDjX/R9cuvObOyuUZhNgBco8i6G4tq1C+aPQPSLl9RmJnNm2rUL5o9A9Lu/7HT\nNYrSx+wHbX/dPFa1x4ett5xujaIK9YtZngHxcxQ2yKpx2W/W65worI9NX/af+qldig8O1yiyc88d\nqsgPiMY/ZKr4I6ZvahTvvPNO2V0wsx5w+PAUzesGZ3VU5J2b3rlbr29qFGed9fFT2o8f/38cPfo+\nVR9HH6z9dfNYnT97AX7+ot9Vv25Z/HdzrjWKvkkUjYPwA+BfUe0vxaDtr5vH8gN8dqrqJ4riC+Au\nZpsVrnFto5frGq5R9ItybtDomxqFmVn/a/VcxvzVV3xFYda25r/mFiz4aEXupGlsdHS07C7YvGt+\nl99c+YrCrG3Nf819+GHzceUq3sVi1glfUZjNu/If/HONwubCVxRm864a7/sxO12+ojArVeOrjaJr\nHq5R2Fw4UZiVqnEB8sMPf9GwvXab7sGOi+pVKKhb7+qJRCHpWkl/I+l1SXeX3R+zcjW/u6VZgpma\neqv0Oon1rsonCklnAH8MXAN8BvhNSZ8ut1dmveYop3OF4iRi0AOJAlgN7ImIfRFxFNgC3FByn8z6\nSKu37DZOIq1qKEUvc7IqXy/c9bQceLNufj+15GFm867xHVutnhspelmrJ47POOMjabits2XWmV5I\nFG0555x/cUrb0aP7+fu/L6EzZlag03vQsfky35LcqV5IFAeA8+vmV6S2k/z85/+jxS5afTFOZ5n3\n1zvHqvr+unmsqu+vm8catP3Ntqy1yr9mXNIC4DXg14G3geeB34yI3aV2zMxsQFT+iiIijku6A9hO\nrfj+gJOEmVn3VP6KwszMytULt8e25IfxaiQ9IGlK0stl96VsklZIekbSTyS9IunLZfepLJLOlLRT\n0kspFhvL7lPZJJ0h6ceSnii7L2WStFfSX6XvxvMt1+3lK4r0MN7r1OoXbwG7gDUR8TeldqwEkn4V\neB/YHBGfK7s/ZZI0BAxFxKSkjwEvAjcM4vcCQNJHIuIXqd73F8CXI6LliaGfSfoKcBlwTkRcX3Z/\nyiLp/wKXRcTh2dbt9SsKP4yXRMSPgFn/hQ+CiDgYEZNp+n1gN7XncQZSREw/THAmtbpk7/46nCNJ\nK4B/Cny/7L5UgGgzB/R6omj0MN7AnhDsVJKGgRFgZ7k9KU8aankJOAg8FRG7yu5Tif4D8G8Z4GRZ\nJ4CnJO2S9LutVuz1RGHWVBp2egy4M11ZDKSI+DAiLqX2DNLnJV1Sdp/KIOmfAVPpalP4ybsrImIV\ntSus29PwdUO9nijaehjPBo+khdSSxEMR8XjZ/amCiPg5sAO4tuy+lOQK4Po0Nv/fgF+TtLnkPpUm\nIt5O/3wH+DNavBqp1xPFLuCTki6QtAhYAwzynQz+lZT9V+DViPhu2R0pk6SPS1qcpn8JuAoYyKJ+\nRHw1Is6PiF+hdq54JiJuKbtfZZD0kXTFjaSPAlcDf91s/Z5OFBFxHJh+GO8nwJZBfRhP0sPAXwIr\nJb0h6XfK7lNZJF0B/DZwZbr178eSBvVX9C8DOyRNUqvTPBkRW0vuk5VvGfCjVLt6DvhhRGxvtnJP\n3x5rZmbzr6evKMzMbP45UZiZWUtOFGZm1pIThZmZteREYWZmLTlRmJlZS04UZmbWkhOFmZm19P8B\ntsAnUVf6tiQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5eb5b44898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Histogram of integrated scores\n",
    "tissues_df.score_integrated.hist(bins=50);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cutoff = 2\n",
    "tissues_df = tissues_df[(tissues_df.score_experiment_unbiased >= cutoff) | (tissues_df.score_integrated >= cutoff)]\n",
    "tissues_df['unbiased'] = (tissues_df.score_experiment_unbiased >= cutoff).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    244315\n",
       "0    209679\n",
       "Name: unbiased, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tissues_df.unbiased.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "453994"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tissues_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uberon_id</th>\n",
       "      <th>uberon_name</th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>gene_symbol</th>\n",
       "      <th>score_text</th>\n",
       "      <th>score_knowledge</th>\n",
       "      <th>score_experiment</th>\n",
       "      <th>score_experiment_unbiased</th>\n",
       "      <th>score_integrated</th>\n",
       "      <th>unbiased</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>uterine cervix</td>\n",
       "      <td>60</td>\n",
       "      <td>ACTB</td>\n",
       "      <td>1.6</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4.9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>uterine cervix</td>\n",
       "      <td>142</td>\n",
       "      <td>PARP1</td>\n",
       "      <td>2.2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>uterine cervix</td>\n",
       "      <td>207</td>\n",
       "      <td>AKT1</td>\n",
       "      <td>2.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>uterine cervix</td>\n",
       "      <td>218</td>\n",
       "      <td>ALDH3A1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>uterine cervix</td>\n",
       "      <td>226</td>\n",
       "      <td>ALDOA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4.8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         uberon_id     uberon_name  entrez_gene_id gene_symbol  score_text  \\\n",
       "26  UBERON:0000002  uterine cervix              60        ACTB         1.6   \n",
       "60  UBERON:0000002  uterine cervix             142       PARP1         2.2   \n",
       "84  UBERON:0000002  uterine cervix             207        AKT1         2.1   \n",
       "90  UBERON:0000002  uterine cervix             218     ALDH3A1         0.6   \n",
       "96  UBERON:0000002  uterine cervix             226       ALDOA         0.0   \n",
       "\n",
       "    score_knowledge  score_experiment  score_experiment_unbiased  \\\n",
       "26                0                 4                          4   \n",
       "60                0                 0                          0   \n",
       "84                0                 0                          0   \n",
       "90                4                 0                          0   \n",
       "96                4                 4                          4   \n",
       "\n",
       "    score_integrated  unbiased  \n",
       "26               4.9         1  \n",
       "60               2.2         0  \n",
       "84               2.6         0  \n",
       "90               4.2         0  \n",
       "96               4.8         1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tissues_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uberon_id</th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>unbiased</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>TISSUES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>142</td>\n",
       "      <td>0</td>\n",
       "      <td>TISSUES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>207</td>\n",
       "      <td>0</td>\n",
       "      <td>TISSUES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>218</td>\n",
       "      <td>0</td>\n",
       "      <td>TISSUES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>UBERON:0000002</td>\n",
       "      <td>226</td>\n",
       "      <td>1</td>\n",
       "      <td>TISSUES</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         uberon_id  entrez_gene_id  unbiased   source\n",
       "26  UBERON:0000002              60         1  TISSUES\n",
       "60  UBERON:0000002             142         0  TISSUES\n",
       "84  UBERON:0000002             207         0  TISSUES\n",
       "90  UBERON:0000002             218         0  TISSUES\n",
       "96  UBERON:0000002             226         1  TISSUES"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tissues_df = tissues_df[['uberon_id', 'entrez_gene_id', 'unbiased']]\n",
    "tissues_df['source'] = 'TISSUES'\n",
    "tissues_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Bgee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>CL:0000015</th>\n",
       "      <th>CL:0000019</th>\n",
       "      <th>CL:0000023</th>\n",
       "      <th>CL:0000115</th>\n",
       "      <th>CL:0000169</th>\n",
       "      <th>CL:0000738</th>\n",
       "      <th>CL:0000842</th>\n",
       "      <th>CL:0002092</th>\n",
       "      <th>CL:0002328</th>\n",
       "      <th>...</th>\n",
       "      <th>UBERON:0008188</th>\n",
       "      <th>UBERON:0009834</th>\n",
       "      <th>UBERON:0009835</th>\n",
       "      <th>UBERON:0010414</th>\n",
       "      <th>UBERON:0011876</th>\n",
       "      <th>UBERON:0013540</th>\n",
       "      <th>UBERON:0013541</th>\n",
       "      <th>UBERON:0013554</th>\n",
       "      <th>UBERON:0014892</th>\n",
       "      <th>UBERON:0016525</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 189 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   entrez_gene_id  CL:0000015  CL:0000019  CL:0000023  CL:0000115  CL:0000169  \\\n",
       "0               1           0           0           0           0           0   \n",
       "1               2           1           0           0           1           0   \n",
       "\n",
       "   CL:0000738  CL:0000842  CL:0002092  CL:0002328       ...        \\\n",
       "0           0           0           0           0       ...         \n",
       "1           1           0           1           1       ...         \n",
       "\n",
       "   UBERON:0008188  UBERON:0009834  UBERON:0009835  UBERON:0010414  \\\n",
       "0               0               0               0               0   \n",
       "1               1               1               1               1   \n",
       "\n",
       "   UBERON:0011876  UBERON:0013540  UBERON:0013541  UBERON:0013554  \\\n",
       "0               0               0               0               0   \n",
       "1               1               1               0               1   \n",
       "\n",
       "   UBERON:0014892  UBERON:0016525  \n",
       "0               0               0  \n",
       "1               1               0  \n",
       "\n",
       "[2 rows x 189 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "commit = '08ba54e83ee8e28dec22b4351d29e23f1d034d30'\n",
    "url = utils.rawgit('dhimmel', 'bgee', commit, 'data/present-in-adult.tsv.gz')\n",
    "bgee_df = pandas.read_table(url, compression='gzip')\n",
    "bgee_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uberon_id</th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>unbiased</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CL:0000015</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>Bgee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CL:0000015</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>Bgee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>CL:0000015</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>Bgee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>CL:0000015</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>Bgee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>CL:0000015</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>Bgee</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     uberon_id  entrez_gene_id  unbiased source\n",
       "1   CL:0000015               2         1   Bgee\n",
       "8   CL:0000015              16         1   Bgee\n",
       "14  CL:0000015              23         1   Bgee\n",
       "27  CL:0000015              37         1   Bgee\n",
       "28  CL:0000015              38         1   Bgee"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bgee_df = pandas.melt(bgee_df, id_vars='entrez_gene_id', var_name='uberon_id', value_name='present')\n",
    "bgee_df = bgee_df.query('present == 1')\n",
    "bgee_df['unbiased'] = 1\n",
    "bgee_df = bgee_df.rename(columns={'GeneID': 'entrez_gene_id'})\n",
    "bgee_df = bgee_df[['uberon_id', 'entrez_gene_id', 'unbiased']]\n",
    "bgee_df['source'] = 'Bgee'\n",
    "bgee_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1150688"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bgee_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concatenate Bgee and TISSUES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def condense(df):\n",
    "    \"\"\"Consolidate relationships into a unique anatomy-gene observations.\"\"\"\n",
    "    row = pandas.Series()\n",
    "    sources = set(df['source'])\n",
    "    row['sources'] = '|'.join(sorted(sources))\n",
    "    row['unbiased'] = df.unbiased.max()\n",
    "    if sources == {'TISSUES'}:\n",
    "        row['license'] = 'CC BY 4.0'\n",
    "    else:\n",
    "        row['license'] = None\n",
    "    return row\n",
    "\n",
    "concat_df = pandas.concat([tissues_df, bgee_df])\n",
    "concat_df = concat_df.groupby(['uberon_id', 'entrez_gene_id']).apply(condense).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    1353158\n",
       "0     178467\n",
       "Name: unbiased, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat_df.unbiased.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with gzip.open('AeG-expression.tsv.gz', 'wt') as write_file:\n",
    "    concat_df.to_csv(write_file, sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}