compile/DaG-association.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "import pandas\n",
    "\n",
    "# local imports\n",
    "sys.path.insert(0, '../')\n",
    "import utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read DO Slim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_id</th>\n",
       "      <th>doid_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:2531</td>\n",
       "      <td>Hematologic cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:1319</td>\n",
       "      <td>Brain cancer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     doid_id           doid_name\n",
       "0  DOID:2531  Hematologic cancer\n",
       "1  DOID:1319        Brain cancer"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "commit = '72614ade9f1cc5a5317b8f6836e1e464b31d5587'\n",
    "url = utils.rawgit('dhimmel', 'disease-ontology', commit, 'data/slim-terms.tsv')\n",
    "disease_df = pandas.read_table(url)\n",
    "disease_df = disease_df.rename(columns={'doid': 'doid_id', 'name': 'doid_name'})\n",
    "disease_df = disease_df[['doid_id', 'doid_name']]\n",
    "disease_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Entrez Gene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>gene_symbol</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>A2M</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   entrez_gene_id gene_symbol\n",
       "0               1        A1BG\n",
       "1               2         A2M"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "commit = '6e133f9ef8ce51a4c5387e58a6cc97564a66cec8'\n",
    "url = utils.rawgit('dhimmel', 'entrez-gene', commit, 'data/genes-human.tsv')\n",
    "gene_df = pandas.read_table(url)\n",
    "gene_df = gene_df[gene_df.type_of_gene == 'protein-coding']\n",
    "gene_df = gene_df.rename(columns={'GeneID': 'entrez_gene_id', 'Symbol': 'gene_symbol'})\n",
    "gene_df = gene_df[['entrez_gene_id', 'gene_symbol']]\n",
    "gene_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_id</th>\n",
       "      <th>doid_name</th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>gene_symbol</th>\n",
       "      <th>score_text</th>\n",
       "      <th>score_knowledge</th>\n",
       "      <th>score_cosmic</th>\n",
       "      <th>score_distild</th>\n",
       "      <th>score_integrated_no_distild</th>\n",
       "      <th>score_integrated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:13223</td>\n",
       "      <td>uterine fibroid</td>\n",
       "      <td>60</td>\n",
       "      <td>ACTB</td>\n",
       "      <td>0.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:13223</td>\n",
       "      <td>uterine fibroid</td>\n",
       "      <td>71</td>\n",
       "      <td>ACTG1</td>\n",
       "      <td>0.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      doid_id        doid_name  entrez_gene_id gene_symbol  score_text  \\\n",
       "0  DOID:13223  uterine fibroid              60        ACTB         0.8   \n",
       "1  DOID:13223  uterine fibroid              71       ACTG1         0.8   \n",
       "\n",
       "   score_knowledge  score_cosmic  score_distild  score_integrated_no_distild  \\\n",
       "0              NaN           NaN            NaN                          0.8   \n",
       "1              NaN           NaN            NaN                          0.8   \n",
       "\n",
       "   score_integrated  \n",
       "0               0.8  \n",
       "1               0.8  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# DISEASES\n",
    "commit = 'e0089ef89a56348d7d4e0684a9c51c5747b16237'\n",
    "url = utils.rawgit('dhimmel', 'diseases', commit, 'data/merged-slim.tsv')\n",
    "diseases_df = pandas.read_table(url)\n",
    "diseases_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_id</th>\n",
       "      <th>doid_name</th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>Symbol</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:0001816</td>\n",
       "      <td>angiosarcoma</td>\n",
       "      <td>302</td>\n",
       "      <td>ANXA2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:0001816</td>\n",
       "      <td>angiosarcoma</td>\n",
       "      <td>595</td>\n",
       "      <td>CCND1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DOID:0001816</td>\n",
       "      <td>angiosarcoma</td>\n",
       "      <td>2324</td>\n",
       "      <td>FLT4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        doid_id     doid_name  entrez_gene_id Symbol  count\n",
       "0  DOID:0001816  angiosarcoma             302  ANXA2      1\n",
       "1  DOID:0001816  angiosarcoma             595  CCND1      1\n",
       "2  DOID:0001816  angiosarcoma            2324   FLT4      1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# DOAF\n",
    "commit = 'bbe1c326aa385416e36d02b144e89e2b99e700b6'\n",
    "url = utils.rawgit('dhimmel', 'doaf', commit, 'data/doaf.tsv')\n",
    "doaf_df = pandas.read_table(url)\n",
    "doaf_df = doaf_df.rename(columns={'doid_code': 'doid_id', 'GeneID': 'entrez_gene_id'})\n",
    "doaf_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_id</th>\n",
       "      <th>doid_name</th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>geneSymbol</th>\n",
       "      <th>count</th>\n",
       "      <th>pubmeds_max</th>\n",
       "      <th>score_max</th>\n",
       "      <th>score_mean</th>\n",
       "      <th>associationType</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:0050156</td>\n",
       "      <td>idiopathic pulmonary fibrosis</td>\n",
       "      <td>729238</td>\n",
       "      <td>SFTPA2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.620284</td>\n",
       "      <td>0.620284</td>\n",
       "      <td>Biomarker|GeneticVariation</td>\n",
       "      <td>BeFree|CLINVAR|CTD_human|UNIPROT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:0050156</td>\n",
       "      <td>idiopathic pulmonary fibrosis</td>\n",
       "      <td>7015</td>\n",
       "      <td>TERT</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0.422153</td>\n",
       "      <td>0.422153</td>\n",
       "      <td>Biomarker|GeneticVariation</td>\n",
       "      <td>BeFree|CLINVAR|CTD_human|GAD|LHGDN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        doid_id                      doid_name  entrez_gene_id geneSymbol  \\\n",
       "0  DOID:0050156  idiopathic pulmonary fibrosis          729238     SFTPA2   \n",
       "1  DOID:0050156  idiopathic pulmonary fibrosis            7015       TERT   \n",
       "\n",
       "   count  pubmeds_max  score_max  score_mean             associationType  \\\n",
       "0      1            1   0.620284    0.620284  Biomarker|GeneticVariation   \n",
       "1      1           10   0.422153    0.422153  Biomarker|GeneticVariation   \n",
       "\n",
       "                               source  \n",
       "0    BeFree|CLINVAR|CTD_human|UNIPROT  \n",
       "1  BeFree|CLINVAR|CTD_human|GAD|LHGDN  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# DisGeNET\n",
    "commit = 'fdc5f42f2da745cbf71d7b4cc5021de5685e4a11'\n",
    "url = utils.rawgit('dhimmel', 'disgenet', commit, 'data/consolidated.tsv')\n",
    "disgenet_df = pandas.read_table(url)\n",
    "disgenet_df = disgenet_df.rename(columns={'doid_code': 'doid_id', 'geneId': 'entrez_gene_id'})\n",
    "disgenet_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_id</th>\n",
       "      <th>doid_name</th>\n",
       "      <th>locus</th>\n",
       "      <th>high_confidence</th>\n",
       "      <th>primary</th>\n",
       "      <th>status</th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>symbol</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:9970</td>\n",
       "      <td>obesity</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>HC-P</td>\n",
       "      <td>3953</td>\n",
       "      <td>LEPR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:9970</td>\n",
       "      <td>obesity</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>HC-P</td>\n",
       "      <td>4094</td>\n",
       "      <td>MAF</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     doid_id doid_name  locus  high_confidence  primary status  \\\n",
       "0  DOID:9970   obesity      0                1        1   HC-P   \n",
       "1  DOID:9970   obesity     14                1        1   HC-P   \n",
       "\n",
       "   entrez_gene_id symbol  \n",
       "0            3953   LEPR  \n",
       "1            4094    MAF  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hetio GWAS\n",
    "commit = '0617ea7ea8268f21f5ca1b8dbe487dd12671fc7b'\n",
    "url = utils.rawgit('dhimmel', 'gwas-catalog', commit, 'data/gene-associations.tsv')\n",
    "gwas_df = pandas.read_table(url)\n",
    "gwas_df = gwas_df.rename(columns={'doid_code': 'doid_id', 'gene': 'entrez_gene_id'})\n",
    "gwas_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "diseases_df = diseases_df.query('score_integrated_no_distild >= 2')\n",
    "doaf_df = doaf_df.query('count >= 3')\n",
    "disgenet_df = disgenet_df.query('score_max >= 0.06')\n",
    "gwas_df = gwas_df[gwas_df.status == 'HC-P']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "diseases_df['provenance'] = 'DISEASES'\n",
    "doaf_df['provenance'] = 'DOAF'\n",
    "disgenet_df['provenance'] = 'DisGeNET'\n",
    "gwas_df['provenance'] = 'GWAS Catalog'\n",
    "\n",
    "diseases_df['license'] = 'CC BY 4.0'\n",
    "doaf_df['license'] = ''\n",
    "disgenet_df['license'] = 'ODbL 1.0'\n",
    "gwas_df['license'] = 'CC BY 4.0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DisGeNET        7552\n",
       "DISEASES        4990\n",
       "DOAF            1649\n",
       "GWAS Catalog    1284\n",
       "Name: provenance, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs = [df[['doid_id', 'entrez_gene_id', 'provenance', 'license']]\n",
    "       for df in (diseases_df, doaf_df, disgenet_df, gwas_df)]\n",
    "concat_df = pandas.concat(dfs)\n",
    "concat_df = disease_df.merge(gene_df.merge(concat_df))\n",
    "concat_df.provenance.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_id</th>\n",
       "      <th>doid_name</th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>gene_symbol</th>\n",
       "      <th>sources</th>\n",
       "      <th>license</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:2531</td>\n",
       "      <td>Hematologic cancer</td>\n",
       "      <td>25</td>\n",
       "      <td>ABL1</td>\n",
       "      <td>DISEASES|DisGeNET</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:2531</td>\n",
       "      <td>Hematologic cancer</td>\n",
       "      <td>27</td>\n",
       "      <td>ABL2</td>\n",
       "      <td>DisGeNET</td>\n",
       "      <td>ODbL 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DOID:2531</td>\n",
       "      <td>Hematologic cancer</td>\n",
       "      <td>54</td>\n",
       "      <td>ACP5</td>\n",
       "      <td>DISEASES</td>\n",
       "      <td>CC BY 4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DOID:2531</td>\n",
       "      <td>Hematologic cancer</td>\n",
       "      <td>113</td>\n",
       "      <td>ADCY7</td>\n",
       "      <td>DisGeNET</td>\n",
       "      <td>ODbL 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DOID:2531</td>\n",
       "      <td>Hematologic cancer</td>\n",
       "      <td>142</td>\n",
       "      <td>PARP1</td>\n",
       "      <td>DISEASES|DisGeNET</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     doid_id           doid_name  entrez_gene_id gene_symbol  \\\n",
       "0  DOID:2531  Hematologic cancer              25        ABL1   \n",
       "1  DOID:2531  Hematologic cancer              27        ABL2   \n",
       "2  DOID:2531  Hematologic cancer              54        ACP5   \n",
       "3  DOID:2531  Hematologic cancer             113       ADCY7   \n",
       "4  DOID:2531  Hematologic cancer             142       PARP1   \n",
       "\n",
       "             sources    license  \n",
       "0  DISEASES|DisGeNET       None  \n",
       "1           DisGeNET   ODbL 1.0  \n",
       "2           DISEASES  CC BY 4.0  \n",
       "3           DisGeNET   ODbL 1.0  \n",
       "4  DISEASES|DisGeNET       None  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def condense(df):\n",
    "    \"\"\"Consolidate multiple associations into a single Series.\"\"\"\n",
    "    row = pandas.Series()\n",
    "    row['sources'] = '|'.join(df.provenance)\n",
    "    licenses = set(df.license)\n",
    "    licenses.discard('')\n",
    "    try:\n",
    "        row['license'], = licenses\n",
    "    except ValueError:\n",
    "        row['license'] = None\n",
    "    return row\n",
    "\n",
    "short_df = concat_df.groupby(['doid_id', 'entrez_gene_id']).apply(condense).reset_index()\n",
    "short_df = disease_df.merge(gene_df.merge(short_df))\n",
    "short_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "short_df.to_csv('DaG-association.tsv', sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}