debugged extract pubmed through search

chufangao · Oct 29, 2024 · e6679d1 · e6679d1
1 parent cc0a957
commit e6679d1
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/llm_prediction_on_pubmed/extract_pubmed_abstracts_through_search.py b/llm_prediction_on_pubmed/extract_pubmed_abstracts_through_search.py
@@ -87,10 +87,15 @@ def search_and_extract_pubmed(data_path,NCBI_api_key,email = ''):
                 reference_list = existing_reference_dict['References']
                 #get PMID from existing references
                 existing_pmids = [reference_list[i]['PMID'] for i in range(len(reference_list))]
+                existing_reference_types = [reference_list[i]['Reference type'].lower() for i in range(len(reference_list))]
+                if 'result' in existing_reference_types or 'search_result' in existing_reference_types:
+                    continue
                 trial_ref_exists_in_data = True
 
             if trial_ref_exists_in_data:
-                pmids = pmids - existing_pmids
+                # filter out existing pmids
+                pmids = [pmid for pmid in pmids if pmid not in existing_pmids]
+
             if len(pmids) == 0:
                 continue
             else:

diff --git a/pipeline.sh b/pipeline.sh
@@ -12,8 +12,8 @@ SAVE_PATH=/srv/local/data/CTO
 echo "Getting LLM predictions on Pubmed data"
 cd llm_prediction_on_pubmed 
 
-echo "Extracting and Updating Pubmed data"
-python extract_pubmed_abstracts.py --data_path $DATA_PATH --save_path $SAVE_PATH #--dev 
+# echo "Extracting and Updating Pubmed data"
+# python extract_pubmed_abstracts.py --data_path $DATA_PATH --save_path $SAVE_PATH #--dev 
 echo "Search Pubmed and extract abstracts"
 python extract_pubmed_abstracts_through_search.py --data_path $DATA_PATH --save_path $SAVE_PATH #--dev
 # echo "Retrieving top 2 relevant abstracts"