Add streamflow diags (#126)

docs/source/parameters.rst

@@ -140,8 +140,10 @@ types and default values for the parameters. ::
         # The following parameters aren't defined in `default.ini`
         # Path to observation data for diurnal cycle runs
         dc_obs_climo = string
-        # Path to observation data for time-series-required ("enso_diags","qbo","area_mean_time_series") runs
+        # Path to observation data for most time-series-required ("enso_diags","qbo","area_mean_time_series") runs
         obs_ts = string
+	# Path to observation data for streamflow diagnostics
+	streamflow_obs_ts = string
         # Start year for the reference data
         ref_start_yr = string
         # End year (i.e., the last year to use) for the reference data

docs/source/post.mysimulation.cfg

@@ -63,6 +63,14 @@ years = "1:100:10",
   mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
   vars = "FSH,RH2M"
 
+  [[ rof_monthly ]]
+  input_subdir = "archive/rof/hist"
+  input_files = "mosart.h0"
+  frequency = "monthly"
+  mapping_file = ""
+  vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"
+  extra_vars = 'areatotal2'
+
 # E3SM Diags every 20 and 50 years
 [e3sm_diags]
 active = True
@@ -79,6 +87,8 @@ ts_num_years = 10
   ref_start_yr = 1979
   # This needs to be set for QBO Diags
   ref_final_yr = 2016
+  # This needs to be set for streamflow diags
+  streamflow_obs_ts = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/time-series/'
   # These two lines need to be included if diurnal cycle diagnostics are being run.
   dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
   climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"

docs/source/tutorial.rst

@@ -7,7 +7,7 @@ Tutorial
 These examples are for use on LCRC machines (e.g., Chrysalis). Some parameters
 must be changed for use on other machines. These include the paths for
 ``input``, ``output``, ``www``, ``mapping_file``, ``reference_data_path``, ``obs_ts``,
-and ``dc_obs_climo``. Different machines also have different partition names, so
+``streamflow_obs_ts``, and ``dc_obs_climo``. Different machines also have different partition names, so
 ``partition`` may need to be changed as well.
 
 

tests/integration/test_bash_generation.cfg

@@ -59,6 +59,14 @@ years = "1:100:10",
   mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
   vars = "FSH,RH2M"
 
+  [[ rof_monthly ]]
+  input_subdir = "archive/rof/hist"
+  input_files = "mosart.h0"
+  frequency = "monthly"
+  mapping_file = ""
+  vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"
+  extra_vars = 'areatotal2'
+
 [glb]
 active = False
 
@@ -75,6 +83,7 @@ ts_num_years = 10
   obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series'
   dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
   climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
+  streamflow_obs_ts = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/time-series/'
 
 [e3sm_diags_vs_model]
 active = False

tests/integration/test_complete_run.cfg

@@ -59,11 +59,21 @@ years = "1:100:10",
   mapping_file = /home/ac.zender/data/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
   vars = "FSH,RH2M"
 
+  [[ rof_monthly ]]
+  input_subdir = "archive/rof/hist"
+  input_files = "mosart.h0"
+  frequency = "monthly"
+  mapping_file = ""
+  vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"
+  extra_vars = 'areatotal2'
+
 [e3sm_diags]
 active = True
 years = "1:20:20", "1:50:50",
 ts_num_years = 10
 # Currently using default `sets`
+# All sets need to use latest E3SM Diags because of acme_diags to e3sm_diags name change.
+environment_commands="source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_env_dev_20210827"
 
   [[ atm_monthly_180x360_aave ]]
   short_name = '20210122.v2_test01.piControl.ne30pg2_EC30to60E2r2-1900_ICG.chrysalis'
@@ -72,6 +82,7 @@ ts_num_years = 10
   obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series'
   dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
   climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
+  streamflow_obs_ts = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/time-series/'
 
 [e3sm_diags_vs_model]
 active = False

tests/integration/test_environment_commands.cfg

@@ -37,7 +37,7 @@ years = "1:20:20", "1:50:50",
 ts_num_years = 10
 # Just testing one set using alternative environment
 sets = "qbo",
-environment_commands="source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_env_dev"
+environment_commands="source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_env_dev_20210827"
 
   [[ atm_monthly_180x360_aave ]]
   short_name = '20210122.v2_test01.piControl.ne30pg2_EC30to60E2r2-1900_ICG.chrysalis'

zppy/e3sm_diags.py

@@ -81,6 +81,14 @@ def e3sm_diags(config, scriptDir):
                             % (sub, start_yr, end_yr, c["ts_num_years"]),
                         )
                     )
+                    if "streamflow" in c["sets"]:
+                        dependencies.append(
+                            os.path.join(
+                                scriptDir,
+                                "ts_rof_monthly_%04d-%04d-%04d.status"
+                                % (start_yr, end_yr, c["ts_num_years"]),
+                            )
+                        )
 
             if not c["dry_run"]:
                 # Submit job

zppy/templates/default.ini

@@ -76,7 +76,7 @@ walltime = string(default="02:00:00")
 reference_data_path = string(default="")
 short_name = string(default="")
 cfg = string(default="")
-sets = string_list(default=list("lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean"))
+sets = string_list(default=list("lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow"))
 backend = string(default="mpl")
 diff_title = string(default="Model - Observations")
 run_type = string(default="model_vs_obs")

zppy/templates/e3sm_diags.bash

@@ -88,6 +88,16 @@ done
 cd ..
 {%- endif %}
 
+{%- if "streamflow" in sets %}
+rofDir="{{ output }}/post/rof/native/ts/monthly/{{ ts_num_years }}yr"
+mkdir -p rof_links
+cd rof_links
+v="RIVER_DISCHARGE_OVER_LAND_LIQ"
+xml_name=${v}_${Y1}01_${Y2}12.xml
+cdscan -x ${xml_name} ${rofDir}/${v}_*.nc
+cd ..
+{%- endif %}
+
 # Run E3SM Diags
 echo
 echo ===== RUN E3SM DIAGS model_vs_obs =====
@@ -98,19 +108,23 @@ cat > e3sm.py << EOF
 import os
 import numpy
 {%- if "area_mean_time_series" in sets %}
-from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
+from e3sm_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
 {%- endif %}
-from acme_diags.parameter.core_parameter import CoreParameter
+from e3sm_diags.parameter.core_parameter import CoreParameter
 {%- if "diurnal_cycle" in sets %}
-from acme_diags.parameter.diurnal_cycle_parameter import DiurnalCycleParameter
+from e3sm_diags.parameter.diurnal_cycle_parameter import DiurnalCycleParameter
 {%- endif %}
 {%- if "enso_diags" in sets %}
-from acme_diags.parameter.enso_diags_parameter import EnsoDiagsParameter
+from e3sm_diags.parameter.enso_diags_parameter import EnsoDiagsParameter
 {%- endif %}
 {%- if "qbo" in sets %}
-from acme_diags.parameter.qbo_parameter import QboParameter
+from e3sm_diags.parameter.qbo_parameter import QboParameter
+{%- endif %}
+{%- if "streamflow" in sets %}
+from e3sm_diags.parameter.streamflow_parameter import StreamflowParameter
 {%- endif %}
-from acme_diags.run import runner
+
+from e3sm_diags.run import runner
 
 short_name = '${short}'
 test_ts = 'ts_links'
@@ -194,6 +208,18 @@ dc_param.normalize_test_amp = False
 params.append(dc_param)
 {%- endif %}
 
+{%- if "streamflow" in sets %}
+streamflow_param = StreamflowParameter()
+streamflow_param.reference_data_path = '{{ streamflow_obs_ts }}'
+streamflow_param.test_data_path = 'rof_links'
+streamflow_param.test_name = short_name
+streamflow_param.test_start_yr = start_yr
+streamflow_param.test_end_yr = end_yr # Streamflow gauge station data range from year 1986 to 1995
+streamflow_param.ref_start_yr = "1986"
+streamflow_param.ref_end_yr = "1995"
+params.append(streamflow_param)
+{%- endif %}
+
 # Run
 runner.sets_to_run = {{ sets }}
 runner.run_diags(params)

zppy/templates/e3sm_diags_vs_model.bash

@@ -68,8 +68,8 @@ echo
 cat > e3sm.py << EOF
 import os
 import numpy
-from acme_diags.parameter.core_parameter import CoreParameter
-from acme_diags.run import runner
+from e3sm_diags.parameter.core_parameter import CoreParameter
+from e3sm_diags.run import runner
 
 param = CoreParameter()