changes in public data search and helper function

R/external.bold.public.search.R

@@ -101,35 +101,26 @@ bold.public.search <- function(taxonomy = NULL,
 
       generate.batch.ids = generate.batches(trial_query_input,batch.size = 5)
 
-
       result.pre.filter = lapply(generate.batch.ids,
                                  function(x) fetch.public.data(x))
 
-      # removing empty results
-
-      result.post.filter = Filter(function(df) nrow(df) > 0,
-                                  result.pre.filter)
 
       # Binding the list of dataframes
 
-      result=result.post.filter%>%
+      result=result.pre.filter%>%
         bind_rows(.)
       }
 
   }
 
-  if(nrow(result)==0)
-
-  {
-    stop("Data could not be retrieved. Please re-check the parameters.")
+  if(nrow(result)==0) stop("Data could not be retrieved. Please re-check the parameters.")
 
-  }
+  if(nrow(result)>1050000) warning("Data cap of 1 million records reached. All records might not have been retrieved. Please rephrase the search")
 
   result = result%>%
     dplyr::select(processid,
                   sampleid)
 
-
   return(result)
 
 }

R/helper.public.search.public_data.R

@@ -118,7 +118,7 @@ fetch.public.data<-function (query)
 
   if (get.data.query$status_code==422)
 
-   stop ("Query limit exceeded. Please reduce the number of search terms")
+    stop ("Query limit exceeded. Please reduce the number of search terms")
 
   # Convert the data into text
 
@@ -140,26 +140,26 @@ fetch.public.data<-function (query)
                            "/download?format=tsv",
                            sep="")
 
-  suppressWarnings(final_data<-data.table::fread(url_download_data,
-                                                 sep = '\t',
-                                                 quote = "",
-                                                 data.table = FALSE,
-                                                 showProgress = F)%>%
-                     data.frame())
+
+  temp_file <- tempfile()
+
+  suppressWarnings(download_data<-download.file(url_download_data,
+                               destfile = temp_file,
+                               quiet = TRUE))
+
+  final_data<-read.delim(temp_file,
+                         sep='\t')
 
   # Some of the column data types are reassigned
 
-  # final_data.pre=reassign.data.type(final_data)
+  final_data=reassign.data.type(final_data)
+
+  final_data=final_data[,intersect(names(final_data),bold.fields.info()$field)]
 
   final_data$collection_date_start<-as.Date(final_data$collection_date_start,format("%Y-%m-%d"))
 
   final_data$collection_date_end<-as.Date(final_data$collection_date_end,format("%Y-%m-%d"))
 
-  final_data_preset=check_and_return_preset_df(final_data,
-                             category = "check_return",
-                             preset = 'public.data.fields')
-
-  return(final_data_preset)
+  return(final_data)
 
 }
-