Change input file formats and input paths in chilled

message_ix_buildings/chilled/core/climate.py

@@ -44,19 +44,25 @@
 from message_ix_buildings.chilled.preprocess.message_raster import (
     create_message_raster,  # type: ignore
 )
-from message_ix_buildings.chilled.util.config import Config  # type: ignore
-from message_ix_buildings.chilled.util.util import (
+from message_ix_buildings.chilled.util.base import (
     get_archs,
-    get_logger,
+    get_paths,
     load_all_scenarios_data,
     load_parametric_analysis_data,
 )
+from message_ix_buildings.chilled.util.common import get_logger
+from message_ix_buildings.chilled.util.config import Config  # type: ignore
 
 log = get_logger(__name__)
 
 
 def create_climate_variables_maps(config: "Config", start_time: datetime.datetime):
-    out_path = os.path.join(config.project_path, "out", "version", config.vstr)
+    project_path = get_paths(config, "project_path")
+    dle_path = get_paths(config, "dle_path")
+    isimip_bias_adj_path = get_paths(config, "isimip_bias_adj_path")
+    isimip_ewembi_path = get_paths(config, "isimip_ewembi_path")
+
+    out_path = os.path.join(project_path, "out", "version", config.vstr)
     archetype_path = os.path.join(out_path, "rasters")
     save_path = os.path.join(out_path, "VDD_ene_calcs")
 
@@ -105,10 +111,10 @@ def map_calculated_variables(args):
         endstr = ".nc"
 
         if str(clim) == "hist":
-            isi_folder = config.isimip_ewemib_path
+            isi_folder = isimip_ewembi_path
             filestr = climate_filestr_hist
         else:
-            isi_folder = config.isimip_bias_adj_path
+            isi_folder = isimip_bias_adj_path
             filestr = climate_filestr_future
 
         filepath = os.path.join(
@@ -135,12 +141,12 @@ def map_calculated_variables(args):
         t_oa_gbm = t_out_ave.groupby("time.month")
 
         i_sol_v = xr.open_dataarray(
-            os.path.join(config.dle_path, "EWEMBI_vert_irrad_1980-2009_avg.nc")
+            os.path.join(dle_path, "EWEMBI_vert_irrad_1980-2009_avg.nc")
         )  # Values  in daily Wh/m2
 
         # Horizontal irradiation
         i_sol_h = xr.open_dataarray(
-            os.path.join(config.dle_path, "EWEMBI_horiz_irrad_1980-2009_avg.nc")
+            os.path.join(dle_path, "EWEMBI_horiz_irrad_1980-2009_avg.nc")
         )  # Values in daily Wh/m2
 
         if config.arch_setting == "regional":
@@ -756,7 +762,8 @@ def read_netcdf_files(input_args):
 
 
 def aggregate_urban_rural_files(config: "Config"):
-    out_path = os.path.join(config.project_path, "out", "version", config.vstr)
+    project_path = get_paths(config, "project_path")
+    out_path = os.path.join(project_path, "out", "version", config.vstr)
     save_path = os.path.join(out_path, "VDD_ene_calcs")
 
     output_path_vdd = os.path.join(
@@ -834,7 +841,8 @@ def aggregate_urban_rural_files(config: "Config"):
 
 
 def make_vdd_total_maps(config: "Config"):
-    out_path = os.path.join(config.project_path, "out", "version", config.vstr)
+    project_path = get_paths(config, "project_path")
+    out_path = os.path.join(project_path, "out", "version", config.vstr)
     save_path = os.path.join(out_path, "VDD_ene_calcs")
 
     output_path_vdd = os.path.join(
@@ -1133,7 +1141,9 @@ def make_map(
 
 
 def process_construction_shares(config: "Config"):
-    out_path = os.path.join(config.project_path, "out", "version", config.vstr)
+    project_path = get_paths(config, "project_path")
+    dle_path = get_paths(config, "dle_path")
+    out_path = os.path.join(project_path, "out", "version", config.vstr)
     floorarea_path = os.path.join(out_path, "floorarea_country")
 
     output_path = os.path.join(
@@ -1153,7 +1163,7 @@ def process_construction_shares(config: "Config"):
 
     # If constr_setting == 1, then process construction shares. Otherwise, skip
     if config.constr_setting == 1:
-        input_path = config.dle_path
+        input_path = dle_path
 
         dsc = xr.Dataset()
         for urt in config.urts:
@@ -1226,8 +1236,10 @@ def process_construction_shares(config: "Config"):
 
 
 def process_floor_area_maps(config: "Config"):
-    input_path = config.dle_path
-    out_path = os.path.join(config.project_path, "out", "version", config.vstr)
+    project_path = get_paths(config, "project_path")
+    dle_path = get_paths(config, "dle_path")
+    input_path = dle_path
+    out_path = os.path.join(project_path, "out", "version", config.vstr)
     save_path = os.path.join(out_path, "floorarea_country")
 
     output_path = os.path.join(
@@ -1354,8 +1366,11 @@ def process_floor_area_maps(config: "Config"):
 
 
 def process_country_maps(config: "Config"):
-    input_path = config.dle_path
-    out_path = os.path.join(config.project_path, "out", "version", config.vstr)
+    project_path = get_paths(config, "project_path")
+    dle_path = get_paths(config, "dle_path")
+
+    input_path = dle_path
+    out_path = os.path.join(project_path, "out", "version", config.vstr)
     save_path = os.path.join(out_path, "floorarea_country")
 
     output_path = os.path.join(
@@ -1436,8 +1451,11 @@ def process_country_maps(config: "Config"):
 
 
 def process_final_maps(config: "Config"):
-    input_path = config.dle_path
-    out_path = os.path.join(config.project_path, "out", "version", config.vstr)
+    project_path = get_paths(config, "project_path")
+    dle_path = get_paths(config, "dle_path")
+
+    input_path = dle_path
+    out_path = os.path.join(project_path, "out", "version", config.vstr)
     vdd_path = os.path.join(
         out_path,
         "VDD_ene_calcs",
@@ -1835,8 +1853,11 @@ def process_final_maps(config: "Config"):
 def process_iso_tables(config: "Config"):
     start = datetime.datetime.now()
 
-    input_path = config.dle_path
-    out_path = os.path.join(config.project_path, "out", "version", config.vstr)
+    project_path = get_paths(config, "project_path")
+    dle_path = get_paths(config, "dle_path")
+
+    input_path = dle_path
+    out_path = os.path.join(project_path, "out", "version", config.vstr)
     vdd_path = os.path.join(out_path, "VDD_ene_calcs", config.gcm, config.rcp)
     floorarea_path = os.path.join(out_path, "floorarea_country", config.gcm, config.rcp)
     finalmaps_path = os.path.join(out_path, "final_maps", config.gcm, config.rcp)
@@ -2140,7 +2161,11 @@ def aggregate_ncfile(args: tuple) -> xr.Dataset:
 
 
 def create_climate_outputs(config: "Config", start_time: datetime.datetime):
-    out_path = os.path.join(config.project_path, "out", "version", config.vstr)
+    project_path = get_paths(config, "project_path")
+    dle_path = get_paths(config, "dle_path")
+    isimip_bias_adj_path = get_paths(config, "isimip_bias_adj_path")
+    isimip_ewemib_path = get_paths(config, "isimip_ewemib_path")
+    out_path = os.path.join(project_path, "out", "version", config.vstr)
     archetype_path = os.path.join(out_path, "rasters")
     save_path = os.path.join(out_path, "VDD_ene_calcs")
 
@@ -2180,10 +2205,10 @@ def create_climate_outputs(config: "Config", start_time: datetime.datetime):
         nyrs_clim = int(years_clim[1]) - int(years_clim[0]) + 1
 
         if str(clim) == "hist":
-            isi_folder = config.isimip_ewemib_path
+            isi_folder = isimip_ewemib_path
             filestr = config.climate_filestr_hist
         else:
-            isi_folder = config.isimip_bias_adj_path
+            isi_folder = isimip_bias_adj_path
             filestr = config.climate_filestr_future
 
         filepath = os.path.join(
@@ -2226,12 +2251,12 @@ def create_climate_outputs(config: "Config", start_time: datetime.datetime):
         # Vertical irradiation
         # i_sol_v = xr.open_dataarray(input_folder+'CERES_vert_irrad_2001-13_avg.nc') #Values  in daily Wh/m2
         i_sol_v = xr.open_dataarray(
-            os.path.join(config.dle_path, "EWEMBI_vert_irrad_1980-2009_avg.nc")
+            os.path.join(dle_path, "EWEMBI_vert_irrad_1980-2009_avg.nc")
         )  # Values  in daily Wh/m2
 
         # Horizontal irradiation
         i_sol_h = xr.open_dataarray(
-            os.path.join(config.dle_path, "EWEMBI_horiz_irrad_1980-2009_avg.nc")
+            os.path.join(dle_path, "EWEMBI_horiz_irrad_1980-2009_avg.nc")
         )  # Values in daily Wh/m2
 
         # i_sol = i_sol.sel(time=slice(years_clim[0],years_clim[1]))

message_ix_buildings/chilled/functions/regression.py

@@ -4,7 +4,7 @@
 import pyam  # type: ignore
 import statsmodels.formula.api as smf  # type: ignore
 
-from message_ix_buildings.chilled.util.util import get_logger
+from message_ix_buildings.chilled.util.common import get_logger
 
 log = get_logger(__name__)
 

message_ix_buildings/chilled/postprocess/postprocess.py

@@ -2,11 +2,13 @@
 
 import pandas as pd
 
+from message_ix_buildings.chilled.util.base import get_paths
+from message_ix_buildings.chilled.util.common import get_logger
 from message_ix_buildings.chilled.util.config import Config
-from message_ix_buildings.chilled.util.util import get_logger
 
 log = get_logger(__name__)
-cfg = Config()
+cfg = Config(user="MEAS")
+dle_path = get_paths(cfg, "dle_path")
 
 
 def get_sturm_data(input_path, input_version_name):
@@ -196,6 +198,14 @@ def postprocess_electricity_demand(input_path, input_version_name):
     )
 
     # Save files
+
+    df_chilled.to_csv(
+        os.path.join(version_output_path, "chilled_postprocessed.csv"),
+        index=False,
+    )
+
+    log.info("Saved: " + os.path.join(version_output_path, "chilled_postprocessed.csv"))
+
     df_sturm.to_csv(
         os.path.join(version_output_path, "sturm_building_stock_inputs.csv"),
         index=False,
@@ -254,4 +264,4 @@ def postprocess_electricity_demand(input_path, input_version_name):
     )
 
 
-postprocess_electricity_demand(cfg.dle_path, cfg.vstr)
+postprocess_electricity_demand(dle_path, cfg.vstr2 + "_" + cfg.vstr)

message_ix_buildings/chilled/preprocess/archetypes.py

@@ -11,19 +11,22 @@
 from functions.variable_dicts import VARS_ARCHETYPES  # type: ignore
 from preprocess.message_raster import create_message_raster  # type: ignore
 
-from message_ix_buildings.chilled.util.config import Config  # type: ignore
-from message_ix_buildings.chilled.util.util import (
+from message_ix_buildings.chilled.util.base import (
     get_archs,
-    get_logger,
+    get_paths,
     read_arch_inputs_df,
     read_arch_reg_df,
 )
+from message_ix_buildings.chilled.util.common import get_logger
+from message_ix_buildings.chilled.util.config import Config  # type: ignore
 
 log = get_logger(__name__)
 
 
 def create_archetypes(config: "Config"):
-    out_path = os.path.join(config.project_path, "out", "version")
+    project_path = get_paths(config, "dle_path")
+
+    out_path = os.path.join(project_path, "out", "version")
     archetype_path = os.path.join(out_path, config.vstr, "rasters")
 
     # if archetypes folder does not exist, create it
@@ -49,7 +52,7 @@ def create_archetypes(config: "Config"):
     )
 
     for arch in vers_archs:
-        arch_reg = read_arch_reg_df(config, arch)
+        arch_reg = read_arch_reg_df(config).query("arch == @arch")
 
         # Create map of archetypes based on MESSAGE regions raster
         arch_map = xr.Dataset(
@@ -101,7 +104,9 @@ def create_archetypes(config: "Config"):
 
 
 def create_archetype_variables(config: "Config"):
-    out_path = os.path.join(config.project_path, "out", "version")
+    project_path = get_paths(config, "dle_path")
+
+    out_path = os.path.join(project_path, "out", "version")
     archetype_path = os.path.join(out_path, config.vstr, "rasters")
 
     # get archs
@@ -115,7 +120,7 @@ def map_archetype_variables(args):
         )
 
         # read in input files
-        arch_inputs = read_arch_inputs_df(config, arch)
+        arch_inputs = read_arch_inputs_df(config).query("arch == @arch")
 
         # read in relevant archetype raster
         map = xr.open_dataset(

message_ix_buildings/chilled/preprocess/message_raster.py

@@ -9,6 +9,7 @@
 import pandas as pd
 import xarray as xr
 
+from message_ix_buildings.chilled.util.base import get_paths
 from message_ix_buildings.chilled.util.config import Config
 
 
@@ -35,11 +36,12 @@ def create_message_raster(config: "Config"):
         ISO attributes in a dataframe
 
     """
-    input_path = os.path.join(config.dle_path)
+    input_path = get_paths(config, "dle_path")
+    message_region_file = get_paths(config, "message_region_map_file")
 
     if config.node == "R11":
         msgregions = pd.read_excel(
-            config.message_region_file,
+            message_region_file,
             sheet_name="regional definition",
         )
 

message_ix_buildings/chilled/run_agg.py

@@ -10,8 +10,8 @@
     process_floor_area_maps,
     process_iso_tables,
 )
+from message_ix_buildings.chilled.util.common import get_logger
 from message_ix_buildings.chilled.util.config import Config  # type: ignore
-from message_ix_buildings.chilled.util.util import get_logger
 
 log = get_logger(__name__)
 

message_ix_buildings/chilled/run_main.py

@@ -14,8 +14,8 @@
     process_floor_area_maps,
     process_iso_tables,
 )
+from message_ix_buildings.chilled.util.common import get_logger
 from message_ix_buildings.chilled.util.config import Config  # type: ignore
-from message_ix_buildings.chilled.util.util import get_logger
 
 log = get_logger(__name__)
 

message_ix_buildings/chilled/run_preprocess.py

@@ -5,8 +5,8 @@
     create_archetype_variables,
     create_archetypes,
 )
+from message_ix_buildings.chilled.util.common import get_logger
 from message_ix_buildings.chilled.util.config import Config
-from message_ix_buildings.chilled.util.util import get_logger
 
 log = get_logger(__name__)