Convert Excel files to CSV - Cardiometabolic disorders (#1570)

* convert to .csv file format

* use read_csv_files method to read the newly converted csv files

* deleting info.csv files as we are not using them in the model

src/scripts/cardio_metabolic_disorders_analyses/cardiometabolicdisorders_plots.py

@@ -23,6 +23,7 @@
     simplified_births,
     symptommanager,
 )
+from tlo.util import read_csv_files
 
 # %%
 resourcefilepath = Path("./resources")
@@ -189,7 +190,7 @@ def convert_output(output_path):
     )
 
     # get prevalence + lower and upper values
-    prev_range = pd.read_excel(resourcefilepath / "cmd" / "ResourceFile_cmd_condition_prevalence.xlsx", sheet_name=None)
+    prev_range = read_csv_files(resourcefilepath / "cmd" / "ResourceFile_cmd_condition_prevalence", files=None)
     baseline_error = [(prev_range[f'{condition}']['value'].values - prev_range[f'{condition}']['lower'].values),
                       (prev_range[f'{condition}']['upper'].values - prev_range[f'{condition}']['value'].values)]
     if 'gbd_value' in prev_range[f'{condition}']:
@@ -494,10 +495,10 @@ def make_incidence_plot(condition, type):
     condition_title = condition_title.title()
 
     if type == 'incidence':
-        inc_range = pd.read_excel(resourcefilepath / "cmd" / f"ResourceFile_cmd_condition_and_events_{type}.xlsx",
-                                  sheet_name=None)
+        inc_range = read_csv_files(resourcefilepath / "cmd" / f"ResourceFile_cmd_condition_and_events_{type}",
+                                  files=None)
     else:
-        inc_range = pd.read_excel(resourcefilepath / "cmd" / f"ResourceFile_cmd_event_{type}.xlsx", sheet_name=None)
+        inc_range = read_csv_files(resourcefilepath / "cmd" / f"ResourceFile_cmd_event_{type}", files=None)
     asymptomatic_error = [(inc_range[f'{condition}']['value'].values - inc_range[f'{condition}']['lower'].values),
                           (inc_range[f'{condition}']['upper'].values - inc_range[f'{condition}']['value'].values)]
 

src/scripts/schistosomiasis/schisto_calibration_check.py

@@ -23,6 +23,7 @@
     simplified_births,
     symptommanager,
 )
+from tlo.util import read_csv_files
 
 resourcefilepath = Path("./resources")
 outputpath = Path("./outputs")
@@ -105,8 +106,8 @@ def get_model_prevalence_by_district(spec: str):
 
 def get_expected_prevalence_by_district(species: str):
     """Get the prevalence of a particular species from the data (which is for year 2010/2011)."""
-    expected_district_prevalence = pd.read_excel(resourcefilepath / 'ResourceFile_Schisto.xlsx',
-                                                 sheet_name='District_Params_' + species.lower())
+    expected_district_prevalence = read_csv_files(resourcefilepath / 'ResourceFile_Schisto',
+                                                 files='District_Params_' + species.lower())
     expected_district_prevalence.set_index("District", inplace=True)
     expected_district_prevalence = expected_district_prevalence.loc[:, 'Prevalence'].to_dict()
     return expected_district_prevalence

src/tlo/methods/cardio_metabolic_disorders.py

@@ -30,7 +30,7 @@
 from tlo.methods.hsi_event import HSI_Event
 from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
 from tlo.methods.symptommanager import Symptom
-from tlo.util import random_date
+from tlo.util import random_date, read_csv_files
 
 if TYPE_CHECKING:
     from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
@@ -259,32 +259,33 @@ def __init__(self, name=None, resourcefilepath=None, do_log_df: bool = False, do
     def read_parameters(self, data_folder):
         """Read parameter values from files for condition onset, removal, deaths, and initial prevalence.
 
-        ResourceFile_cmd_condition_onset.xlsx = parameters for onset of conditions
-        ResourceFile_cmd_condition_removal.xlsx  = parameters for removal of conditions
-        ResourceFile_cmd_condition_death.xlsx  = parameters for death rate from conditions
-        ResourceFile_cmd_condition_prevalence.xlsx  = initial and target prevalence for conditions
-        ResourceFile_cmd_condition_symptoms.xlsx  = symptoms for conditions
-        ResourceFile_cmd_condition_hsi.xlsx  = HSI parameters for conditions
-        ResourceFile_cmd_condition_testing.xlsx  = community testing parameters for conditions (currently only
-        hypertension)
-        ResourceFile_cmd_events.xlsx  = parameters for occurrence of events
-        ResourceFile_cmd_events_death.xlsx  = parameters for death rate from events
-        ResourceFile_cmd_events_symptoms.xlsx  = symptoms for events
-        ResourceFile_cmd_events_hsi.xlsx  = HSI parameters for events
+        Folders
+            ResourceFile_cmd_condition_onset = parameters for onset of conditions
+            ResourceFile_cmd_condition_removal  = parameters for removal of conditions
+            ResourceFile_cmd_condition_death  = parameters for death rate from conditions
+            ResourceFile_cmd_condition_prevalence  = initial and target prevalence for conditions
+            ResourceFile_cmd_condition_symptoms  = symptoms for conditions
+            ResourceFile_cmd_condition_hsi  = HSI parameters for conditions
+            ResourceFile_cmd_condition_testing  = community testing parameters for conditions (currently only
+            hypertension)
+            ResourceFile_cmd_events  = parameters for occurrence of events
+            ResourceFile_cmd_events_death  = parameters for death rate from events
+            ResourceFile_cmd_events_symptoms  = symptoms for events
+            ResourceFile_cmd_events_hsi  = HSI parameters for events
 
         """
         cmd_path = Path(self.resourcefilepath) / "cmd"
-        cond_onset = pd.read_excel(cmd_path / "ResourceFile_cmd_condition_onset.xlsx", sheet_name=None)
-        cond_removal = pd.read_excel(cmd_path / "ResourceFile_cmd_condition_removal.xlsx", sheet_name=None)
-        cond_death = pd.read_excel(cmd_path / "ResourceFile_cmd_condition_death.xlsx", sheet_name=None)
-        cond_prevalence = pd.read_excel(cmd_path / "ResourceFile_cmd_condition_prevalence.xlsx", sheet_name=None)
-        cond_symptoms = pd.read_excel(cmd_path / "ResourceFile_cmd_condition_symptoms.xlsx", sheet_name=None)
-        cond_hsi = pd.read_excel(cmd_path / "ResourceFile_cmd_condition_hsi.xlsx", sheet_name=None)
-        cond_testing = pd.read_excel(cmd_path / "ResourceFile_cmd_condition_testing.xlsx", sheet_name=None)
-        events_onset = pd.read_excel(cmd_path / "ResourceFile_cmd_events.xlsx", sheet_name=None)
-        events_death = pd.read_excel(cmd_path / "ResourceFile_cmd_events_death.xlsx", sheet_name=None)
-        events_symptoms = pd.read_excel(cmd_path / "ResourceFile_cmd_events_symptoms.xlsx", sheet_name=None)
-        events_hsi = pd.read_excel(cmd_path / "ResourceFile_cmd_events_hsi.xlsx", sheet_name=None)
+        cond_onset = read_csv_files(cmd_path / "ResourceFile_cmd_condition_onset", files=None)
+        cond_removal = read_csv_files(cmd_path / "ResourceFile_cmd_condition_removal", files=None)
+        cond_death = read_csv_files(cmd_path / "ResourceFile_cmd_condition_death", files=None)
+        cond_prevalence = read_csv_files(cmd_path / "ResourceFile_cmd_condition_prevalence", files=None)
+        cond_symptoms = read_csv_files(cmd_path / "ResourceFile_cmd_condition_symptoms", files=None)
+        cond_hsi = read_csv_files(cmd_path / "ResourceFile_cmd_condition_hsi", files=None)
+        cond_testing = read_csv_files(cmd_path / "ResourceFile_cmd_condition_testing", files=None)
+        events_onset = read_csv_files(cmd_path / "ResourceFile_cmd_events", files=None)
+        events_death = read_csv_files(cmd_path / "ResourceFile_cmd_events_death", files=None)
+        events_symptoms = read_csv_files(cmd_path / "ResourceFile_cmd_events_symptoms", files=None)
+        events_hsi = read_csv_files(cmd_path / "ResourceFile_cmd_events_hsi", files=None)
 
         self.load_parameters_from_dataframe(pd.read_csv(cmd_path / "ResourceFile_cmd_parameters.csv"))