BertNERChemprotLargeScale.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chemprot: Bert NER on Pubmed Abstracts using PyTorch Prediction\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "import logging\n",
    "\n",
    "sys.path.append(\"src\")\n",
    "\n",
    "logging.basicConfig(level=\"INFO\", handlers=[logging.StreamHandler(sys.stdout)],\n",
    "                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_job = \"chemprot-ner-bert-2022-07-23-21-56-34-969\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "local_temp=\"temp\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf $local_temp\n",
    "!mkdir -p $local_temp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bucket and role set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "s3_uri_pubmedjson = \"s3://aegovan-data/pubmed-json/pubmed19n06\"\n",
    "\n",
    "s3_output_base = \"s3://aegovan-data/chemprotnerlargescale/\"\n",
    "s3_code_path = \"s3://aegovan-data/chemprotnercode\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-07-24 19:40:52,238 - botocore.credentials - INFO - Found credentials in shared credentials file: ~/.aws/credentials\n",
      "2022-07-24 19:40:52,378 - botocore.credentials - INFO - Found credentials in shared credentials file: ~/.aws/credentials\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "\n",
    "#from sagemaker import get_execution_role\n",
    "\n",
    "import sagemaker.session \n",
    "sm_session = sagemaker.session.Session()\n",
    "account_id =  boto3.client('sts').get_caller_identity().get('Account')\n",
    "\n",
    "# role=get_execution_role()\n",
    "role =\"arn:aws:iam::{}:role/service-role/AmazonSageMaker-ExecutionRole-20181222T162635\".format(account_id)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict\n",
    "\n",
    "This shows you how to train BERT on SageMaker using SPOT instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance_type = \"ml.p3.2xlarge\"\n",
    "instance_count=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2022-07-23 22:40:11 Starting - Preparing the instances for training\n",
      "2022-07-23 22:40:11 Downloading - Downloading input data\n",
      "2022-07-23 22:40:11 Training - Training image download completed. Training in progress.\n",
      "2022-07-23 22:40:11 Uploading - Uploading generated training model\n",
      "2022-07-23 22:40:11 Completed - Training job completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'s3://aegovan-data/chemprotner/output/chemprot-ner-bert-2022-07-23-21-56-34-969/output/model.tar.gz'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sagemaker\n",
    "estimator = sagemaker.estimator.Estimator.attach(training_job)\n",
    "model_uri = estimator.model_data\n",
    "model_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set True if you need spot instance\n",
    "use_spot = False\n",
    "max_run_secs =   5 *24 * 60 * 60\n",
    "spot_wait_sec =  5 * 60\n",
    "max_wait_time_secs = max_run_secs +  spot_wait_sec\n",
    "\n",
    "if not use_spot:\n",
    "    max_wait_time_secs = None\n",
    "    \n",
    "# During local mode, no spot.., use smaller dataset\n",
    "if instance_type == 'local':\n",
    "    use_spot = False\n",
    "    max_wait_time_secs = 0\n",
    "    wait = True\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'chemprot-ner-largescale-20220724194053'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_type = \"chemprot-ner-largescale\"\n",
    "\n",
    "from datetime import datetime\n",
    "job_name = \"{}-{}\".format(job_type,datetime.now().strftime(\"%Y%m%d%H%M%S\"))\n",
    "job_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-07-24 19:40:55,474 - sagemaker.processing - INFO - Uploaded src to s3://aegovan-data/chemprotnercode/chemprot-ner-largescale-20220724194053/source/sourcedir.tar.gz\n",
      "2022-07-24 19:40:55,984 - sagemaker.processing - INFO - runproc.sh uploaded to s3://aegovan-data/chemprotnercode/chemprot-ner-largescale-20220724194053/source/runproc.sh\n",
      "\n",
      "Job Name:  chemprot-ner-largescale-20220724194053\n",
      "Inputs:  [{'InputName': 'input-1', 'AppManaged': False, 'S3Input': {'S3Uri': 's3://aegovan-data/pubmed-json/pubmed19n0650.json', 'LocalPath': '/opt/ml/processing/input/data', 'S3DataType': 'S3Prefix', 'S3InputMode': 'File', 'S3DataDistributionType': 'ShardedByS3Key', 'S3CompressionType': 'None'}}, {'InputName': 'input-2', 'AppManaged': False, 'S3Input': {'S3Uri': 's3://aegovan-data/chemprotner/output/chemprot-ner-bert-2022-07-23-21-56-34-969/output/model.tar.gz', 'LocalPath': '/opt/ml/processing/input/model', 'S3DataType': 'S3Prefix', 'S3InputMode': 'File', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}}, {'InputName': 'code', 'AppManaged': False, 'S3Input': {'S3Uri': 's3://aegovan-data/chemprotnercode/chemprot-ner-largescale-20220724194053/source/sourcedir.tar.gz', 'LocalPath': '/opt/ml/processing/input/code/', 'S3DataType': 'S3Prefix', 'S3InputMode': 'File', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}}, {'InputName': 'entrypoint', 'AppManaged': False, 'S3Input': {'S3Uri': 's3://aegovan-data/chemprotnercode/chemprot-ner-largescale-20220724194053/source/runproc.sh', 'LocalPath': '/opt/ml/processing/input/entrypoint', 'S3DataType': 'S3Prefix', 'S3InputMode': 'File', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}}]\n",
      "Outputs:  [{'OutputName': 'output', 'AppManaged': False, 'S3Output': {'S3Uri': 's3://aegovan-data/chemprotnerlargescale/chemprot-ner-largescale-20220724194053', 'LocalPath': '/opt/ml/processing/output/data', 'S3UploadMode': 'EndOfJob'}}]\n",
      "2022-07-24 19:40:55,991 - sagemaker - INFO - Creating processing-job with name chemprot-ner-largescale-20220724194053\n",
      ".\n",
      "\u001B[34mFound existing installation: typing 3.6.4\u001B[0m\n",
      "\u001B[34mUninstalling typing-3.6.4:\n",
      "  Successfully uninstalled typing-3.6.4\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: torch==1.4.0 in /opt/conda/lib/python3.6/site-packages (from -r requirements.txt (line 1)) (1.4.0)\u001B[0m\n",
      "\u001B[34mCollecting transformers==3.0.1\n",
      "  Downloading transformers-3.0.1-py3-none-any.whl (757 kB)\u001B[0m\n",
      "\u001B[34mCollecting scikit-learn==0.23.1\n",
      "  Downloading scikit_learn-0.23.1-cp36-cp36m-manylinux1_x86_64.whl (6.8 MB)\u001B[0m\n",
      "\u001B[34mCollecting seqeval==0.0.17\n",
      "  Downloading seqeval-0.0.17.tar.gz (20 kB)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: numpy>=1.13.3 in /opt/conda/lib/python3.6/site-packages (from scikit-learn==0.23.1->-r requirements.txt (line 3)) (1.16.4)\u001B[0m\n",
      "\u001B[34mCollecting threadpoolctl>=2.0.0\n",
      "  Downloading threadpoolctl-3.1.0-py3-none-any.whl (14 kB)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.6/site-packages (from scikit-learn==0.23.1->-r requirements.txt (line 3)) (1.2.2)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.6/site-packages (from scikit-learn==0.23.1->-r requirements.txt (line 3)) (1.0.1)\u001B[0m\n",
      "\u001B[34mCollecting Keras>=2.2.4\n",
      "  Downloading keras-2.9.0-py2.py3-none-any.whl (1.6 MB)\u001B[0m\n",
      "\u001B[34mCollecting sentencepiece!=0.1.92\n",
      "  Downloading sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\u001B[0m\n",
      "\u001B[34mCollecting sacremoses\n",
      "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.6/site-packages (from transformers==3.0.1->-r requirements.txt (line 2)) (4.56.0)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: dataclasses in /opt/conda/lib/python3.6/site-packages (from transformers==3.0.1->-r requirements.txt (line 2)) (0.8)\u001B[0m\n",
      "\u001B[34mCollecting tokenizers==0.8.0-rc4\n",
      "  Downloading tokenizers-0.8.0rc4-cp36-cp36m-manylinux1_x86_64.whl (3.0 MB)\u001B[0m\n",
      "\u001B[34mCollecting regex!=2019.12.17\n",
      "  Downloading regex-2022.7.24-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (751 kB)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: packaging in /opt/conda/lib/python3.6/site-packages (from transformers==3.0.1->-r requirements.txt (line 2)) (20.9)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from transformers==3.0.1->-r requirements.txt (line 2)) (2.22.0)\u001B[0m\n",
      "\u001B[34mCollecting filelock\n",
      "  Downloading filelock-3.4.1-py3-none-any.whl (9.9 kB)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.6/site-packages (from packaging->transformers==3.0.1->-r requirements.txt (line 2)) (2.4.7)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==3.0.1->-r requirements.txt (line 2)) (3.0.4)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==3.0.1->-r requirements.txt (line 2)) (1.25.11)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==3.0.1->-r requirements.txt (line 2)) (2.8)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==3.0.1->-r requirements.txt (line 2)) (2020.12.5)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==3.0.1->-r requirements.txt (line 2)) (1.15.0)\u001B[0m\n",
      "\u001B[34mRequirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==3.0.1->-r requirements.txt (line 2)) (7.1.2)\u001B[0m\n",
      "\u001B[34mBuilding wheels for collected packages: seqeval, sacremoses\n",
      "  Building wheel for seqeval (setup.py): started\u001B[0m\n",
      "\u001B[34m  Building wheel for seqeval (setup.py): finished with status 'done'\n",
      "  Created wheel for seqeval: filename=seqeval-0.0.17-py3-none-any.whl size=7637 sha256=af7bac24223690f881950f5c137248db0289b1f7d7526e09f82587c3a6c5013d\n",
      "  Stored in directory: /root/.cache/pip/wheels/a0/8f/33/b734af42c4a11d13fa4412f5019cde239430c25ca654f29536\n",
      "  Building wheel for sacremoses (setup.py): started\n",
      "  Building wheel for sacremoses (setup.py): finished with status 'done'\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895254 sha256=a15b1c6ed4f1507efee427d335ecc7d058ccf8c81dccbe222c097ea03abf708b\n",
      "  Stored in directory: /root/.cache/pip/wheels/4c/64/31/e9900a234b23fb3e9dc565d6114a9d6ff84a72dbdd356502b4\u001B[0m\n",
      "\u001B[34mSuccessfully built seqeval sacremoses\u001B[0m\n",
      "\u001B[34mInstalling collected packages: regex, tokenizers, threadpoolctl, sentencepiece, sacremoses, Keras, filelock, transformers, seqeval, scikit-learn\u001B[0m\n",
      "\u001B[34m  Attempting uninstall: scikit-learn\n",
      "    Found existing installation: scikit-learn 0.21.2\n",
      "    Uninstalling scikit-learn-0.21.2:\n",
      "      Successfully uninstalled scikit-learn-0.21.2\u001B[0m\n",
      "\u001B[34mSuccessfully installed Keras-2.9.0 filelock-3.4.1 regex-2022.7.24 sacremoses-0.0.53 scikit-learn-0.23.1 sentencepiece-0.1.96 seqeval-0.0.17 threadpoolctl-3.1.0 tokenizers-0.8.0rc4 transformers-3.0.1\u001B[0m\n",
      "\u001B[34m{'inputdatadir': '/opt/ml/processing/input/data', 'modeltar': '/opt/ml/processing/input/model/model.tar.gz', 'outputdatadir': '/opt/ml/processing/output/data', 'batchsize': 8, 'log_level': 'INFO'}\u001B[0m\n",
      "\u001B[34m2022-07-25 02:46:04,680 - transformers.configuration_utils - INFO - loading configuration file /opt/ml/processing/input/model/artifacts/config.json\u001B[0m\n",
      "\u001B[34m2022-07-25 02:46:04,681 - transformers.configuration_utils - INFO - Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForTokenClassification\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\",\n",
      "    \"3\": \"LABEL_3\",\n",
      "    \"4\": \"LABEL_4\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2,\n",
      "    \"LABEL_3\": 3,\n",
      "    \"LABEL_4\": 4\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 28996\u001B[0m\n",
      "\u001B[34m}\u001B[0m\n",
      "\u001B[34m2022-07-25 02:46:04,681 - transformers.modeling_utils - INFO - loading weights file /opt/ml/processing/input/model/artifacts/pytorch_model.bin\u001B[0m\n",
      "\u001B[34m2022-07-25 02:46:07,564 - transformers.modeling_utils - INFO - All model checkpoint weights were used when initializing BertForTokenClassification.\u001B[0m\n",
      "\u001B[34m2022-07-25 02:46:07,564 - transformers.modeling_utils - INFO - All the weights of BertForTokenClassification were initialized from the model checkpoint at /opt/ml/processing/input/model/artifacts.\u001B[0m\n",
      "\u001B[34mIf your task is similar to the task the model of the ckeckpoint was trained on, you can already use BertForTokenClassification for predictions without further training.\u001B[0m\n",
      "\u001B[34m2022-07-25 02:46:09,225 - __main__ - INFO - Processing file /opt/ml/processing/input/data/pubmed19n0650.json\u001B[0m\n",
      "\u001B[34m2022-07-25 02:59:42,238 - __main__ - INFO - Completed inference file /opt/ml/processing/input/data/pubmed19n0650.json\u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:23,638 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `p1(+/+) w` doesnt line up with entity text `Cyp1(+/+)`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:23,870 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `[2-[[3-(1H-benzimidazo` doesnt line up with entity text `cyclopropanecarboxylic`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:24,619 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `,5,6,7-` doesnt line up with entity text `mercury`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:24,890 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `5-ter` doesnt line up with entity text `[UNK]`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:26,120 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `oxy-b` doesnt line up with entity text `NK(2)`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:26,260 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `-alpha` doesnt line up with entity text `methyl`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:26,997 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `  ` doesnt line up with entity text `Mg`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:27,046 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `thyl)` doesnt line up with entity text `human`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:28,157 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `. cisplat` doesnt line up with entity text `cisplatin`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:28,164 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `thanol ` doesnt line up with entity text `ethanol`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:28,768 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `(11)C]flumazenil ` doesnt line up with entity text `[(11)C]flumazenil`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:29,275 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `lpipe` doesnt line up with entity text `ethyl`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:29,613 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  ` RAN` doesnt line up with entity text `RANK`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:30,425 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `alt ` doesnt line up with entity text `GRK4`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:31,291 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `e α2/` doesnt line up with entity text `α2/α3`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:31,459 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `fect of ` doesnt line up with entity text `TGF-beta`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:31,474 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `ogen bon` doesnt line up with entity text `hydrogen`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:31,608 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `23-hy` doesnt line up with entity text `[UNK]`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:31,753 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `h tacrolim` doesnt line up with entity text `tacrolimus`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:31,785 - bert_ner_position_converter_include_space - WARNING - Something went wrong..position at original text  `lozapine ` doesnt line up with entity text `clozapine`. Skipping the rest of items in this record.. \u001B[0m\n",
      "\u001B[34m2022-07-25 03:00:34,540 - __main__ - INFO - Completed processing file /opt/ml/processing/input/data/pubmed19n0650.json\u001B[0m\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.pytorch.processing import PyTorchProcessor\n",
    "from sagemaker.processing import ProcessingInput, ProcessingOutput\n",
    "\n",
    "processor = PyTorchProcessor( role=role,\n",
    "                    framework_version =\"1.4.0\",\n",
    "                    code_location=s3_code_path,\n",
    "                    instance_count=1,\n",
    "                    instance_type=instance_type,\n",
    "                    max_runtime_in_seconds = max_run_secs,\n",
    "                    volume_size_in_gb=250\n",
    "                    )\n",
    "\n",
    "sm_data_local = '/opt/ml/processing/input/data'\n",
    "sm_model_local = '/opt/ml/processing/input/model'\n",
    "sm_output_local = '/opt/ml/processing/output/data'\n",
    "s3_output_uri = \"{}/{}\".format( s3_output_base.rstrip(\"/\"), job_name)\n",
    "\n",
    "processor.run(\n",
    "    job_name=job_name,\n",
    "    code='chemprot_batch_inference.py',\n",
    "    source_dir='src',\n",
    "    arguments = [\n",
    "        \"--inputdatadir\",sm_data_local,\n",
    "        \"--modeltar\", \"{}/{}\".format(sm_model_local,model_uri.split(\"/\")[-1]),\n",
    "        \"--outputdatadir\",sm_output_local,\n",
    "        \"--batchsize\", \"16\",\n",
    "    ],\n",
    "    inputs=[\n",
    "        ProcessingInput(\n",
    "            source=s3_uri_pubmedjson,\n",
    "            destination=sm_data_local,\n",
    "            s3_data_distribution_type = \"ShardedByS3Key\"\n",
    "        ),\n",
    "    ProcessingInput(\n",
    "            source=model_uri,\n",
    "            destination=sm_model_local,\n",
    "            s3_data_distribution_type = \"FullyReplicated\"\n",
    "        )\n",
    "    ],\n",
    "    outputs=[\n",
    "        ProcessingOutput(output_name='output', \n",
    "                         source=sm_output_local, \n",
    "                         destination=s3_output_uri)\n",
    "            ]\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}