Training verifcation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3_prefix = \"s3://aegovan-data/processed_dataset/train_multiclass.json\"\n",
    "s3_output_prefix = \"{}_trainingdata/\".format(\"/\".join(s3_prefix.split(\"/\")[0:-1]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'s3://aegovan-data/processed_dataset_trainingdata/'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3_output_prefix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "local_temp = \"temp\"\n",
    "local_temp_pred_dir = os.path.join( local_temp, \"pred_results\")\n",
    "local_temp_wk_dir = os.path.join( local_temp, \"wk\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf $local_temp\n",
    "!mkdir -p $local_temp_pred_dir\n",
    "!mkdir -p $local_temp_wk_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import glob\n",
    "from multiprocessing.dummy import Pool as ThreadPool\n",
    "import argparse\n",
    "import datetime \n",
    "import os\n",
    "\n",
    "\n",
    "def upload_file(localpath, s3path):\n",
    "        \"\"\"\n",
    "Uploads a file to s3\n",
    "        :param localpath: The local path\n",
    "        :param s3path: The s3 path in format s3://mybucket/mydir/mysample.txt\n",
    "        \"\"\"\n",
    "\n",
    "        bucket, key = get_bucketname_key(s3path)\n",
    "\n",
    "        if key.endswith(\"/\"):\n",
    "            key = \"{}{}\".format(key, os.path.basename(localpath))\n",
    "        \n",
    "        s3 = boto3.client('s3')\n",
    "        \n",
    "        s3.upload_file(localpath, bucket, key)\n",
    "\n",
    "def get_bucketname_key(uripath):\n",
    "    assert uripath.startswith(\"s3://\")\n",
    "\n",
    "    path_without_scheme = uripath[5:]\n",
    "    bucket_end_index = path_without_scheme.find(\"/\")\n",
    "\n",
    "    bucket_name = path_without_scheme\n",
    "    key = \"/\"\n",
    "    if bucket_end_index > -1:\n",
    "        bucket_name = path_without_scheme[0:bucket_end_index]\n",
    "        key = path_without_scheme[bucket_end_index + 1:]\n",
    "\n",
    "    return bucket_name, key\n",
    "\n",
    "\n",
    "def download_file(s3path, local_dir):\n",
    "    bucket, key = get_bucketname_key(s3path)\n",
    "    \n",
    "    s3 = boto3.client('s3')\n",
    "    \n",
    "    local_file = os.path.join(local_dir, s3path.split(\"/\")[-1])\n",
    "    \n",
    "\n",
    "    s3.download_file(bucket, key, local_file)\n",
    "    \n",
    "def download_object(s3path):\n",
    "    bucket, key = get_bucketname_key(s3path)\n",
    "    \n",
    "    s3 = boto3.client('s3')    \n",
    "\n",
    "    s3_response_object = s3.get_object(Bucket=bucket, Key=key)\n",
    "    object_content = s3_response_object['Body'].read()\n",
    "    \n",
    "    return len(object_content)\n",
    "\n",
    "\n",
    "\n",
    "def list_files(s3path_prefix):\n",
    "    assert s3path_prefix.startswith(\"s3://\")\n",
    "    \n",
    "    bucket, key = get_bucketname_key(s3path_prefix)\n",
    "    \n",
    "   \n",
    "   \n",
    "    s3 = boto3.resource('s3')\n",
    "    \n",
    "    bucket = s3.Bucket(name=bucket)\n",
    "\n",
    "    return ( (o.bucket_name, o.key) for o in bucket.objects.filter(Prefix=key))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def upload_files(local_dir, s3_prefix, num_threads=20):    \n",
    "    input_tuples = ( (f,  s3_prefix) for f in glob.glob(\"{}/*\".format(local_dir)))\n",
    "    \n",
    "    with ThreadPool(num_threads) as pool:\n",
    "        pool.starmap(uploadfile, input_tuples)\n",
    "    \n",
    "\n",
    "\n",
    "def download_files(s3_prefix, local_dir, num_threads=20):    \n",
    "    input_tuples = ( (\"s3://{}/{}\".format(s3_bucket,s3_key),  local_dir) for s3_bucket, s3_key in list_files(s3_prefix))\n",
    "    \n",
    "    with ThreadPool(num_threads) as pool:\n",
    "        results = pool.starmap(download_file, input_tuples)\n",
    "        \n",
    "        \n",
    "\n",
    "def download_objects(s3_prefix, num_threads=20):    \n",
    "    s3_files = ( \"s3://{}/{}\".format(s3_bucket,s3_key) for s3_bucket, s3_key in list_files(s3_prefix))\n",
    "    \n",
    "    with ThreadPool(num_threads) as pool:\n",
    "        results = pool.map(download_object, s3_files)\n",
    "        \n",
    "    return sum(results)/1024\n",
    "        \n",
    "\n",
    "def get_directory_size(start_path):\n",
    "    total_size = 0\n",
    "    for dirpath, dirnames, filenames in os.walk(start_path):\n",
    "        for f in filenames:\n",
    "            fp = os.path.join(dirpath, f)\n",
    "            # skip if it is symbolic link\n",
    "            if not os.path.islink(fp):\n",
    "                total_size += os.path.getsize(fp)\n",
    "    return total_size\n",
    "\n",
    "def get_s3file_size(bucket, key):\n",
    "    s3 = boto3.client('s3')\n",
    "    response = s3.head_object(Bucket=bucket, Key=key)\n",
    "    size = response['ContentLength']\n",
    "    return size\n",
    "    \n",
    "def download_files_min_files(s3_prefix, local_dir, min_file_size=310, num_threads=20):    \n",
    "    input_tuples = ( (\"s3://{}/{}\".format(s3_bucket,s3_key),  local_dir) for s3_bucket, s3_key in list_files(s3_prefix) if get_s3file_size(s3_bucket, s3_key) > min_file_size )\n",
    "    \n",
    "    with ThreadPool(num_threads) as pool:\n",
    "        results = pool.starmap(download_file, input_tuples)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 98.3 ms, sys: 60.1 ms, total: 158 ms\n",
      "Wall time: 1.58 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "download_file(s3_prefix, local_temp_pred_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 aeg  staff  7552394 May  7 22:34 train_multiclass.json\r\n"
     ]
    }
   ],
   "source": [
    "!ls -l $local_temp_pred_dir | grep \"train_multiclass.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_training_full_df = pd.read_json(os.path.join(local_temp_pred_dir, \"train_multiclass.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pubmedId</th>\n",
       "      <th>pubmedabstract</th>\n",
       "      <th>annotations</th>\n",
       "      <th>num_unique_gene_normalised_id</th>\n",
       "      <th>num_gene_normalised_id</th>\n",
       "      <th>normalised_abstract</th>\n",
       "      <th>normalised_abstract_annotations</th>\n",
       "      <th>participant1Id</th>\n",
       "      <th>participant2Id</th>\n",
       "      <th>gene_to_uniprot_map</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17197020</td>\n",
       "      <td>In HEK293 cells, transfected with the Ca2+ cha...</td>\n",
       "      <td>[{'start': '3', 'end': '9', 'name': 'HEK293', ...</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>In HEK293 cells, transfected with the Ca2+ cha...</td>\n",
       "      <td>[{'charOffset': 59, 'len': 6, 'text': 'Q9H1D0'...</td>\n",
       "      <td>Q9H1D0</td>\n",
       "      <td>P18031</td>\n",
       "      <td>{'5464': ['Q15181', 'V9HWB5'], '55503': ['Q9H1...</td>\n",
       "      <td>dephosphorylation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17053781</td>\n",
       "      <td>Receptor interacting protein 140 (RIP140), a l...</td>\n",
       "      <td>[{'start': '0', 'end': '32', 'name': 'Receptor...</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>P48552 (P48552), a ligand-dependent corepresso...</td>\n",
       "      <td>[{'charOffset': 0, 'len': 6, 'text': 'P48552'}...</td>\n",
       "      <td>Q99873</td>\n",
       "      <td>P48552</td>\n",
       "      <td>{'7514': ['O14980', 'B3KWD0'], '8204': ['P4855...</td>\n",
       "      <td>methylation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17079228</td>\n",
       "      <td>TAK1 (transforming growth factor beta-activate...</td>\n",
       "      <td>[{'start': '0', 'end': '4', 'name': 'TAK1', 't...</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>O43318 (O43318) is a serine/threonine kinase t...</td>\n",
       "      <td>[{'charOffset': 0, 'len': 6, 'text': 'O43318'}...</td>\n",
       "      <td>O00743</td>\n",
       "      <td>O43318</td>\n",
       "      <td>{'5524': ['Q15257', 'B4DZF8', 'F6WIT2'], '4216...</td>\n",
       "      <td>dephosphorylation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pubmedId                                     pubmedabstract  \\\n",
       "0  17197020  In HEK293 cells, transfected with the Ca2+ cha...   \n",
       "1  17053781  Receptor interacting protein 140 (RIP140), a l...   \n",
       "2  17079228  TAK1 (transforming growth factor beta-activate...   \n",
       "\n",
       "                                         annotations  \\\n",
       "0  [{'start': '3', 'end': '9', 'name': 'HEK293', ...   \n",
       "1  [{'start': '0', 'end': '32', 'name': 'Receptor...   \n",
       "2  [{'start': '0', 'end': '4', 'name': 'TAK1', 't...   \n",
       "\n",
       "   num_unique_gene_normalised_id  num_gene_normalised_id  \\\n",
       "0                              4                      10   \n",
       "1                              3                      12   \n",
       "2                              5                      24   \n",
       "\n",
       "                                 normalised_abstract  \\\n",
       "0  In HEK293 cells, transfected with the Ca2+ cha...   \n",
       "1  P48552 (P48552), a ligand-dependent corepresso...   \n",
       "2  O43318 (O43318) is a serine/threonine kinase t...   \n",
       "\n",
       "                     normalised_abstract_annotations participant1Id  \\\n",
       "0  [{'charOffset': 59, 'len': 6, 'text': 'Q9H1D0'...         Q9H1D0   \n",
       "1  [{'charOffset': 0, 'len': 6, 'text': 'P48552'}...         Q99873   \n",
       "2  [{'charOffset': 0, 'len': 6, 'text': 'O43318'}...         O00743   \n",
       "\n",
       "  participant2Id                                gene_to_uniprot_map  \\\n",
       "0         P18031  {'5464': ['Q15181', 'V9HWB5'], '55503': ['Q9H1...   \n",
       "1         P48552  {'7514': ['O14980', 'B3KWD0'], '8204': ['P4855...   \n",
       "2         O43318  {'5524': ['Q15257', 'B4DZF8', 'F6WIT2'], '4216...   \n",
       "\n",
       "               class  \n",
       "0  dephosphorylation  \n",
       "1        methylation  \n",
       "2  dephosphorylation  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_training_full_df.head(n=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Rename column so they match predictions, so ground truth can be verified"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_training_full_df = data_training_full_df.rename(\n",
    "    columns={\"pubmedabstract\" : \"abstract\"\n",
    "                                      ,\"annotations\":\"annotations\"\n",
    "                                      }\n",
    "                             \n",
    "                             ) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def missing_uniprot(x):\n",
    "    uniprots =[]\n",
    "    for i in list(x[\"gene_to_uniprot_map\"].values()):\n",
    "        uniprots.extend(i)\n",
    "        \n",
    "    \n",
    "    return x[\"participant1Id\"] not in uniprots or x[\"participant2Id\"] not in uniprots\n",
    "\n",
    "def gene_id_map(x):\n",
    "    gene_to_uniprot_map = x[\"gene_to_uniprot_map\"]\n",
    "    for n, uniprots in x[\"gene_to_uniprot_map\"].items():\n",
    "        if x[\"participant1Id\"]  in uniprots :\n",
    "            gene_to_uniprot_map[n]= x[\"participant1Id\"]\n",
    "        if x[\"participant2Id\"]  in uniprots:\n",
    "            gene_to_uniprot_map[n]= x[\"participant2Id\"]\n",
    "        \n",
    "        \n",
    "    return gene_to_uniprot_map\n",
    "                                    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "                                    \n",
    "data_training_full_df[\"missing_uniprot\"] = data_training_full_df.apply(missing_uniprot, axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: class, dtype: int64)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_training_full_df.query( \"missing_uniprot == True\").groupby(\"class\")[\"class\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "class\n",
       "acetylation             5\n",
       "dephosphorylation      28\n",
       "deubiquitination        2\n",
       "methylation            10\n",
       "other                1116\n",
       "phosphorylation       139\n",
       "ubiquitination          5\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_training_full_df.query( \"missing_uniprot == False\").groupby(\"class\")[\"class\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: class, dtype: int64)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_training_full_df.query(\"participant1Id == participant2Id\").groupby(\"class\")[\"class\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "class\n",
       "acetylation           5\n",
       "dephosphorylation    10\n",
       "deubiquitination      2\n",
       "methylation          10\n",
       "other                10\n",
       "phosphorylation      10\n",
       "ubiquitination        5\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples_subset = data_training_full_df.groupby('class', group_keys=False)\\\n",
    "                .apply(lambda x: x.sample(min(len(x), 10),random_state=45))\n",
    "\n",
    "samples_subset.groupby([\"class\"])[\"class\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pubmedId</th>\n",
       "      <th>abstract</th>\n",
       "      <th>annotations</th>\n",
       "      <th>num_unique_gene_normalised_id</th>\n",
       "      <th>num_gene_normalised_id</th>\n",
       "      <th>normalised_abstract</th>\n",
       "      <th>normalised_abstract_annotations</th>\n",
       "      <th>participant1Id</th>\n",
       "      <th>participant2Id</th>\n",
       "      <th>gene_to_uniprot_map</th>\n",
       "      <th>class</th>\n",
       "      <th>missing_uniprot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>26829474</td>\n",
       "      <td>Faithful segregation of chromosomes in mammali...</td>\n",
       "      <td>[{'start': '39', 'end': '48', 'name': 'mammali...</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>Faithful segregation of chromosomes in mammali...</td>\n",
       "      <td>[{'charOffset': 234, 'len': 6, 'text': 'P06493...</td>\n",
       "      <td>Q96GD4</td>\n",
       "      <td>Q92993</td>\n",
       "      <td>{'10524': ['Q92993', 'A0A024R597', 'A0A024R5E8...</td>\n",
       "      <td>acetylation</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>23441852</td>\n",
       "      <td>The regulation of gene repression by corepress...</td>\n",
       "      <td>[{'start': '214', 'end': '219', 'name': 'Alien...</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>The regulation of gene repression by corepress...</td>\n",
       "      <td>[{'charOffset': 214, 'len': 6, 'text': 'P61201...</td>\n",
       "      <td>Q09472</td>\n",
       "      <td>P61201</td>\n",
       "      <td>{'1387': ['Q92793'], '2033': ['Q09472', 'Q7Z6C...</td>\n",
       "      <td>acetylation</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>20856196</td>\n",
       "      <td>The overexpression of human apurinic/apyrimidi...</td>\n",
       "      <td>[{'start': '22', 'end': '27', 'name': 'human',...</td>\n",
       "      <td>4</td>\n",
       "      <td>27</td>\n",
       "      <td>The overexpression of human P27695 (P27695/P27...</td>\n",
       "      <td>[{'charOffset': 28, 'len': 6, 'text': 'P27695'...</td>\n",
       "      <td>Q09472</td>\n",
       "      <td>P27695</td>\n",
       "      <td>{'4904': ['P67809'], '5243': ['P08183', 'A4D1D...</td>\n",
       "      <td>acetylation</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>22771473</td>\n",
       "      <td>Eukaryotic translation initiation factor 5A (e...</td>\n",
       "      <td>[{'start': '0', 'end': '43', 'name': 'Eukaryot...</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>P63241 (P63241) is a protein subject to hypusi...</td>\n",
       "      <td>[{'charOffset': 0, 'len': 6, 'text': 'P63241'}...</td>\n",
       "      <td>Q92831</td>\n",
       "      <td>P63241</td>\n",
       "      <td>{'10013': ['Q9UBN7', 'Q9BRX7', 'A0A024QZ26', '...</td>\n",
       "      <td>acetylation</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>21157427</td>\n",
       "      <td>SRSF2 is a serine/arginine-rich protein belong...</td>\n",
       "      <td>[{'start': '0', 'end': '5', 'name': 'SRSF2', '...</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>Q01130 is a serine/arginine-rich protein belon...</td>\n",
       "      <td>[{'charOffset': 0, 'len': 6, 'text': 'Q01130'}...</td>\n",
       "      <td>Q01130</td>\n",
       "      <td>Q92993</td>\n",
       "      <td>{'10524': ['Q92993', 'A0A024R597', 'A0A024R5E8...</td>\n",
       "      <td>acetylation</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     pubmedId                                           abstract  \\\n",
       "47   26829474  Faithful segregation of chromosomes in mammali...   \n",
       "150  23441852  The regulation of gene repression by corepress...   \n",
       "34   20856196  The overexpression of human apurinic/apyrimidi...   \n",
       "102  22771473  Eukaryotic translation initiation factor 5A (e...   \n",
       "149  21157427  SRSF2 is a serine/arginine-rich protein belong...   \n",
       "\n",
       "                                           annotations  \\\n",
       "47   [{'start': '39', 'end': '48', 'name': 'mammali...   \n",
       "150  [{'start': '214', 'end': '219', 'name': 'Alien...   \n",
       "34   [{'start': '22', 'end': '27', 'name': 'human',...   \n",
       "102  [{'start': '0', 'end': '43', 'name': 'Eukaryot...   \n",
       "149  [{'start': '0', 'end': '5', 'name': 'SRSF2', '...   \n",
       "\n",
       "     num_unique_gene_normalised_id  num_gene_normalised_id  \\\n",
       "47                               4                      14   \n",
       "150                              4                      11   \n",
       "34                               4                      27   \n",
       "102                              4                      11   \n",
       "149                              6                      13   \n",
       "\n",
       "                                   normalised_abstract  \\\n",
       "47   Faithful segregation of chromosomes in mammali...   \n",
       "150  The regulation of gene repression by corepress...   \n",
       "34   The overexpression of human P27695 (P27695/P27...   \n",
       "102  P63241 (P63241) is a protein subject to hypusi...   \n",
       "149  Q01130 is a serine/arginine-rich protein belon...   \n",
       "\n",
       "                       normalised_abstract_annotations participant1Id  \\\n",
       "47   [{'charOffset': 234, 'len': 6, 'text': 'P06493...         Q96GD4   \n",
       "150  [{'charOffset': 214, 'len': 6, 'text': 'P61201...         Q09472   \n",
       "34   [{'charOffset': 28, 'len': 6, 'text': 'P27695'...         Q09472   \n",
       "102  [{'charOffset': 0, 'len': 6, 'text': 'P63241'}...         Q92831   \n",
       "149  [{'charOffset': 0, 'len': 6, 'text': 'Q01130'}...         Q01130   \n",
       "\n",
       "    participant2Id                                gene_to_uniprot_map  \\\n",
       "47          Q92993  {'10524': ['Q92993', 'A0A024R597', 'A0A024R5E8...   \n",
       "150         P61201  {'1387': ['Q92793'], '2033': ['Q09472', 'Q7Z6C...   \n",
       "34          P27695  {'4904': ['P67809'], '5243': ['P08183', 'A4D1D...   \n",
       "102         P63241  {'10013': ['Q9UBN7', 'Q9BRX7', 'A0A024QZ26', '...   \n",
       "149         Q92993  {'10524': ['Q92993', 'A0A024R597', 'A0A024R5E8...   \n",
       "\n",
       "           class  missing_uniprot  \n",
       "47   acetylation            False  \n",
       "150  acetylation            False  \n",
       "34   acetylation            False  \n",
       "102  acetylation            False  \n",
       "149  acetylation            False  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples_subset.query(\"`class` == 'acetylation'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import json\n",
    "def create_manifest_file(df, outfile):\n",
    "    items = df.to_dict(orient='records' )\n",
    "    with open(outfile , \"w\") as f:\n",
    "        for item in items:\n",
    "            # Write without new lines\n",
    "            item_m  = {}\n",
    "            item_m[\"source\"] = json.dumps(item)\n",
    "            f.write(json.dumps(item_m).replace(\"\\n\", \"\\t\"))\n",
    "            f.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'s3://aegovan-data/processed_dataset_trainingdata/'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " \"{}/\".format(s3_output_prefix.rstrip(\"/\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples_subset_file = \"predictions_sample_subset.json\"\n",
    "samples_subset.to_json(samples_subset_file, orient='records')\n",
    "upload_file(samples_subset_file, \"{}/\".format(s3_output_prefix.rstrip(\"/\")))\n",
    "\n",
    "\n",
    "manifest_file = \"predictions_sample_subset.mainfest\"\n",
    "create_manifest_file(samples_subset, manifest_file)\n",
    "upload_file(manifest_file, \"{}/\".format(s3_output_prefix.rstrip(\"/\")))\n",
    "\n",
    "# Create one manifest file per interaction type\n",
    "s3_manifests = []\n",
    "for i in filter(lambda x: x.lower() != 'other', list(samples_subset[\"class\"].unique())):\n",
    "    manifest_file = \"training_sample_subset_{}.mainfest\".format(i)\n",
    "    create_manifest_file( samples_subset.query(\"`class` == '{}'\".format(i)), manifest_file)\n",
    "    s3_manifest_file = \"{}/{}\".format(s3_output_prefix.rstrip(\"/\"), manifest_file.split(os.path.sep)[-1])\n",
    "    upload_file(manifest_file, s3_manifest_file)\n",
    "    s3_manifests.append(s3_manifest_file)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create sagemaker ground truth labelling job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def create_groundtruth_labelling_job(s3_manifest, s3_gt_output, s3_template, pre_lambda, post_lambda, role, workforce_name, job_name, label_attribute_name=\"class\", workforce_type= \"private-crowd\" ):\n",
    "    client = boto3.client('sagemaker')\n",
    "    \n",
    "    sagemaker_session = sagemaker.Session()\n",
    "    account_id =  boto3.client('sts').get_caller_identity().get('Account')\n",
    "    region = boto3.session.Session().region_name\n",
    "    \n",
    "    workforce_arn = \"arn:aws:sagemaker:{}:{}:workteam/{}/{}\".format(region, account_id, workforce_type, workforce_name)\n",
    "    role_arn = \"arn:aws:iam::{}:role/{}\".format( account_id,  role)\n",
    "    pre_lambda_arn = \"arn:aws:lambda:{}:{}:function:{}\".format(region, account_id,  pre_lambda)\n",
    "    post_lambda_arn = \"arn:aws:lambda:{}:{}:function:{}\".format(region, account_id,  post_lambda)\n",
    "    \n",
    "    num_workers_per_object = 1\n",
    "    task_time_limit_sec = 60  * 60 * 5\n",
    "    task_availablity_sec =60  * 60 * 24 * 10\n",
    "    \n",
    "    job = client.create_labeling_job(LabelingJobName=job_name\n",
    "                                    ,LabelAttributeName = label_attribute_name\n",
    "                                    ,InputConfig = {\n",
    "                                        \"DataSource\": {\n",
    "                                            'S3DataSource': {\n",
    "                                            'ManifestS3Uri': s3_manifest\n",
    "                                            }\n",
    "                                        }\n",
    "                                        \n",
    "                                    }\n",
    "                                  ,OutputConfig={\n",
    "                                        'S3OutputPath': s3_gt_output\n",
    "                                    }\n",
    "\n",
    "                                  , RoleArn = role_arn\n",
    "                                  , HumanTaskConfig={\n",
    "                                    'WorkteamArn': workforce_arn,\n",
    "                                    'UiConfig': {\n",
    "                                        'UiTemplateS3Uri': s3_template\n",
    "                                    },\n",
    "                                    'PreHumanTaskLambdaArn': pre_lambda_arn,\n",
    "                                    'TaskKeywords': [\n",
    "                                        'PPI',\n",
    "                                    ],\n",
    "                                    'TaskTitle': 'Verify PPI extraction for protein {}'.format(s3_manifest.split(\"/\")[-1]),\n",
    "                                    'TaskDescription': 'Verifies PPi extraction',\n",
    "                                    'NumberOfHumanWorkersPerDataObject': num_workers_per_object,\n",
    "                                    'TaskTimeLimitInSeconds': task_time_limit_sec,\n",
    "                                    'TaskAvailabilityLifetimeInSeconds': task_availablity_sec,\n",
    "                                    'MaxConcurrentTaskCount': 10,\n",
    "                                    'AnnotationConsolidationConfig': {\n",
    "                                        'AnnotationConsolidationLambdaArn': post_lambda_arn\n",
    "                                    }\n",
    "                                }\n",
    "                            )\n",
    "    \n",
    "    return job\n",
    "    \n",
    "    \n",
    "\n",
    "def create_groundtruth_labelling_multiple_jobs(lst_s3_manifests, s3_gt_output, s3_template, pre_lambda, post_lambda, role, workforce_name, job_prefix =\"ppi\", label_attribute_name=\"class\"):\n",
    "    job_prefix = \"{}-{}\".format(job_prefix , datetime.now().strftime(\"%Y%m%d%H%M%S\"))\n",
    "    for s3_manifest in lst_s3_manifests:\n",
    "        job_name = \"{}-{}\".format( job_prefix, s3_manifest.split(\"/\")[-1].split(\"_\")[-1].split(\".\")[0])\n",
    "        create_groundtruth_labelling_job(s3_manifest, s3_gt_output, s3_template, pre_lambda, post_lambda, role, workforce_name, job_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "\n",
    "def download_template(template_url):\n",
    "    with urllib.request.urlopen(template_url) as f:\n",
    "        html = f.read().decode('utf-8')\n",
    "\n",
    "    with open(\"template.html\", \"w\") as f:\n",
    "        f.write(html)\n",
    "    \n",
    "download_template('http://raw.githubusercontent.com/elangovana/ppi-sagemaker-groundtruth-verification/main/src/template/template.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "role_name = \"service-role/AmazonSageMaker-ExecutionRole-20210104T161547\"\n",
    "pre_lambda=\"Sagemaker-ppipreprocessing\"\n",
    "post_lambda=\"sagemaker-ppipostprocessing\"\n",
    "s3_gt_output = \"{}_trainingdata_groundtruth/\".format(\"/\".join(s3_prefix.split(\"/\")[0:-1]))\n",
    "workforce_name = \"ppi-team\"\n",
    "s3_template_file = \"{}_gt_templatet/template.html\".format(\"/\".join(s3_prefix.split(\"/\")[0:-1]))\n",
    "\n",
    "upload_file(\"template.html\", s3_template_file )\n",
    "create_groundtruth_labelling_multiple_jobs (s3_manifests,s3_gt_output, s3_template_file,pre_lambda, post_lambda, role_name, workforce_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}