Added filtering to extract results or derived pubmed abstracts for dr…

…ug and biological trials

llm_prediction_on_pubmed/extract_pubmed_abstracts.py

@@ -7,7 +7,7 @@
 import argparse
 from utils import drug_biologics_nct_ids
 import pandas as pd
-
+from dotenv import load_dotenv
 
 
 def get_data(element, source):
@@ -164,13 +164,14 @@ def main(data_path,NCBI_api_key, dev = False):
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('--data_path', type=str, default= None, help='Path to the CITI data folder')
-    parser.add_argument('--NCBI_api_key', type=str, default= None, help='NCBI API key')
+    # parser.add_argument('--NCBI_api_key', type=str, default= None, help='NCBI API key')
     parser.add_argument('--save_path', type=str, default= None, help='Path to save the extracted data')
     parser.add_argument('--dev', action='store_true', help='Run in development mode')
     args = parser.parse_args()
 
     data_path = args.data_path
-    NCBI_api_key = args.NCBI_api_key
+    load_dotenv()
+    NCBI_api_key = os.getenv('NCBI_api_key')
 
     if data_path is None:
         raise ValueError('Please provide the path to the CITI data folder')

pipeline.sh

@@ -1,5 +1,4 @@
 DATA_PATH=/srv/local/data/CTO/CTTI_new
-NCBI_API_KEY='558a8ec64b0df1607941d0261d0a5d273308'
 SAVE_PATH=/srv/local/data/CTO
 
 
@@ -14,7 +13,7 @@ echo "Getting LLM predictions on Pubmed data"
 cd llm_prediction_on_pubmed 
 
 echo "Extracting and Updating Pubmed data"
-python extract_pubmed_abstracts.py --data_path $DATA_PATH --NCBI_api_key $NCBI_API_KEY --save_path $SAVE_PATH #--dev 
+python extract_pubmed_abstracts.py --data_path $DATA_PATH --save_path $SAVE_PATH #--dev 
 # echo "Retrieving top 2 relevant abstracts"
 # python retrieve_top2_abstracts.py --data_path $DATA_PATH --save_path $SAVE_PATH --dev
 # echo "Obtaining LLM predictions"