Feature #2068 ioda2nc v2.0 (#2307)

Co-authored-by: Howard Soh <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: MET Tools Test Account <[email protected]>
dtcenter · Oct 30, 2022 · 1123b41 · 1123b41
1 parent 44c37c2
commit 1123b41
Show file tree

Hide file tree

Showing 19 changed files with 704 additions and 219 deletions.
diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh
@@ -6,7 +6,7 @@ run_unit_tests=false
 run_diff=false
 run_update_truth=false
 met_base_repo=met-base
-met_base_tag=v1.0
+met_base_tag=v1.1
 input_data_version=develop
 truth_data_version=develop
 

diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml
@@ -29,7 +29,7 @@ jobs:
         env:
           SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite
           MET_BASE_REPO: met-base
-          MET_BASE_TAG: v1.0
+          MET_BASE_TAG: v1.1
 
       - name: Push Docker Image
         run: .github/jobs/push_docker_image.sh

diff --git a/data/config/IODA2NCConfig_default b/data/config/IODA2NCConfig_default
@@ -92,7 +92,16 @@ metadata_map = [
    { key = "station_id";   val = "station_id,report_identifier"; },
    { key = "pressure";     val = "air_pressure,pressure"; },
    { key = "height";       val = "height,height_above_mean_sea_level"; },
-   { key = "elevation";    val = ""; }
+   { key = "datetime";     val = "datetime,dateTime"; },
+   { key = "elevation";    val = "elevation,station_elevation"; }
+];
+
+//
+// Default mapping for obs to qc.
+//
+obs_to_qc_map = [
+   { key = "wind_from_direction"; val = "eastward_wind,northward_wind"; },
+   { key = "wind_speed";          val = "eastward_wind,northward_wind"; }
 ];
 
 missing_thresh = [ <=-1e9, >=1e9, ==-9999 ];

diff --git a/docs/Users_Guide/reformat_point.rst b/docs/Users_Guide/reformat_point.rst
@@ -961,13 +961,24 @@ _____________________
 		{ key = "station_id";   val = "station_id,report_identifier"; },
 		{ key = "pressure";     val = "air_pressure,pressure"; },
 		{ key = "height";       val = "height,height_above_mean_sea_level"; },
-		{ key = "elevation";    val = ""; }
+		{ key = "elevation";    val = "elevation,station_elevation"; }
 		];
 
 This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of metadata for IODA data files.
 
 _____________________
 
+.. code-block:: none
+		
+		obs_to_qc_map  = [
+		{ key = "wind_from_direction"; val = "eastward_wind,northward_wind"; },
+		{ key = "wind_speed";          val = "eastward_wind,northward_wind"; }
+		];
+
+This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of QC variable name for IODA data files.
+
+_____________________
+
 .. code-block:: none
 
 		missing_thresh = [ <=-1e9, >=1e9, ==-9999 ];

diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile
@@ -1,5 +1,5 @@
 ARG MET_BASE_REPO=met-base
-ARG MET_BASE_TAG=v1.0
+ARG MET_BASE_TAG=v1.1
 
 FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG}
 MAINTAINER John Halley Gotway <[email protected]>

diff --git a/internal/scripts/docker/Dockerfile.copy b/internal/scripts/docker/Dockerfile.copy
@@ -1,5 +1,5 @@
 ARG MET_BASE_REPO=met-base-unit-test
-ARG MET_BASE_TAG=v1.0
+ARG MET_BASE_TAG=v1.1
 
 FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG}
 MAINTAINER John Halley Gotway <[email protected]>

diff --git a/internal/scripts/environment/development.docker b/internal/scripts/environment/development.docker
@@ -24,7 +24,7 @@ export MET_FREETYPELIB=/usr/lib64
 export MET_JASPERLIB=/usr/lib64
 
 export MET_PYTHON=/usr/bin/python3
-export MET_PYTHON_CC="-I/usr/include/python3.6m -I/usr/include/python3.6m"
+export MET_PYTHON_CC="-I/usr/include/python3.6m"
 export MET_PYTHON_LD="-L/usr/lib64 -lpython3.6m -lpthread -ldl -lutil -lm"
 
 # -D__64BIT__ is required because we've compiled libgrib2c.a with that flag

diff --git a/internal/test_unit/xml/unit_ioda2nc.xml b/internal/test_unit/xml/unit_ioda2nc.xml
@@ -83,4 +83,42 @@
     </output>
   </test>
 
+  <test name="ioda2nc_int_datetime">
+    <exec>&MET_BIN;/ioda2nc</exec>
+    <env>
+      <pair><name>STATION_ID</name>   <value></value></pair>
+      <pair><name>MASK_GRID</name>    <value></value></pair>
+      <pair><name>MASK_POLY</name>    <value></value></pair>
+      <pair><name>MESSAGE_TYPE</name> <value></value></pair>
+    </env>
+    <param> \
+      &DATA_DIR_OBS;/ioda/jopa_satwind_20210701T1200Z_out_0000_reduced.nc4 \
+      &OUTPUT_DIR;/ioda2nc/jopa_satwind_20210701T1200Z_int_datetime.nc \
+      -config &CONFIG_DIR;/IODA2NCConfig_mask \
+      -v 2
+    </param>
+    <output>
+      <point_nc>&OUTPUT_DIR;/ioda2nc/jopa_satwind_20210701T1200Z_int_datetime.nc</point_nc>
+    </output>
+  </test>
+
+  <test name="ioda2nc_v2_string_sid">
+    <exec>&MET_BIN;/ioda2nc</exec>
+    <env>
+      <pair><name>STATION_ID</name>   <value></value></pair>
+      <pair><name>MASK_GRID</name>    <value></value></pair>
+      <pair><name>MASK_POLY</name>    <value></value></pair>
+      <pair><name>MESSAGE_TYPE</name> <value></value></pair>
+    </env>
+    <param> \
+      &DATA_DIR_OBS;/ioda/2021081612_sonde_small.nc \
+      &OUTPUT_DIR;/ioda2nc/2021081612_sonde_small_sid.nc \
+      -config &CONFIG_DIR;/IODA2NCConfig_mask \
+      -v 2
+    </param>
+    <output>
+      <point_nc>&OUTPUT_DIR;/ioda2nc/2021081612_sonde_small_sid.nc</point_nc>
+    </output>
+  </test>
+
 </met_test>
diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h
@@ -650,6 +650,7 @@ static const char conf_key_trunc_factor[]      = "gaussian_trunc_factor";
 static const char conf_key_eclv_points[]       = "eclv_points";
 static const char conf_key_var_name_map[]      = "var_name_map";
 static const char conf_key_metadata_map[]      = "metadata_map";
+static const char conf_key_obs_to_qc_map[]     = "obs_to_qc_map";
 static const char conf_key_missing_thresh[]    = "missing_thresh";
 static const char conf_key_control_id[]        = "control_id";
 static const char conf_key_ens_member_ids[]    = "ens_member_ids";
@@ -897,6 +898,7 @@ static const char conf_key_do_polylines_flag   [] = "do_polylines";
 // PB2NC specific parameter key names
 //
 
+static const char conf_key_datetime[]              = "datetime";
 static const char conf_key_station_id[]            = "station_id";
 static const char conf_key_elevation_range[]       = "elevation_range";
 static const char conf_key_pb_report_type[]        = "pb_report_type";

diff --git a/src/basic/vx_config/config_util.cc b/src/basic/vx_config/config_util.cc
@@ -1099,6 +1099,13 @@ map<ConcatString,ConcatString> parse_conf_obs_name_map(Dictionary *dict) {
 
 ///////////////////////////////////////////////////////////////////////////////
 
+map<ConcatString,StringArray> parse_conf_obs_to_qc_map(Dictionary *dict) {
+   const char *method_name = "parse_conf_obs_to_qc_map() -> ";
+   return parse_conf_key_values_map(dict, conf_key_obs_to_qc_map, method_name);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
 map<ConcatString,UserFunc_1Arg> parse_conf_key_convert_map(
       Dictionary *dict, const char *conf_key_map_name, const char *caller) {
    Dictionary *map_dict = (Dictionary *) 0;

diff --git a/src/basic/vx_config/config_util.h b/src/basic/vx_config/config_util.h
@@ -52,18 +52,21 @@ extern NumArray        parse_conf_eclv_points(Dictionary *dict);
 extern ClimoCDFInfo    parse_conf_climo_cdf(Dictionary *dict);
 extern TimeSummaryInfo parse_conf_time_summary(Dictionary *dict);
 extern std::map<ConcatString,ConcatString> parse_conf_key_value_map(
-                            Dictionary *dict, const char *conf_key_map_name, const char *caller=0);
+                          Dictionary *dict, const char *conf_key_map_name, const char *caller=0);
 extern void            parse_add_conf_key_value_map(
-                            Dictionary *dict, const char *conf_key_map_name, std::map<ConcatString,ConcatString> *m);
+                          Dictionary *dict, const char *conf_key_map_name, std::map<ConcatString,ConcatString> *m);
 extern std::map<ConcatString,ConcatString>
                        parse_conf_message_type_map(Dictionary *dict);
 extern std::map<ConcatString,StringArray>
                        parse_conf_message_type_group_map(Dictionary *dict);
 extern std::map<ConcatString,StringArray> parse_conf_metadata_map(Dictionary *dict);
 extern std::map<ConcatString,ConcatString>
                        parse_conf_obs_name_map(Dictionary *dict);
-extern std::map<ConcatString,UserFunc_1Arg> parse_conf_key_convert_map(
-                            Dictionary *dict, const char *conf_key_map_name, const char *caller=0);
+extern std::map<ConcatString,StringArray>
+                       parse_conf_obs_to_qc_map(Dictionary *dict);
+extern std::map<ConcatString,UserFunc_1Arg>
+                       parse_conf_key_convert_map(
+                          Dictionary *dict, const char *conf_key_map_name, const char *caller=0);
 extern BootInfo        parse_conf_boot(Dictionary *dict);
 extern RegridInfo      parse_conf_regrid(Dictionary *dict, bool error_out = default_dictionary_error_out);
 extern InterpInfo      parse_conf_interp(Dictionary *dict, const char *);

diff --git a/src/libcode/vx_data2d_factory/is_netcdf_file.cc b/src/libcode/vx_data2d_factory/is_netcdf_file.cc
@@ -40,9 +40,6 @@ static const char netcdf_magic  [] = "CDF";
 static const char hdf_magic     [] = "HDF";
 static const int  netcdf_magic_len = m_strlen(netcdf_magic);
 
-static const string nccf_att_name   = "Conventions";
-static const string nccf_att_name_l = "conventions";
-static const string nccf_att_name_U = "CONVENTIONS";
 static const string nccf_att_value  = "CF-";
 static const string nccf_att_value2 = "CF ";
 static const string nccf_att_value3 = "COARDS";
@@ -96,9 +93,7 @@ bool is_nccf_file(const char * filename)
       NcFile *nc_file = open_ncfile(filename);
 
       if (!IS_INVALID_NC_P(nc_file)) {
-         bool found = get_global_att(nc_file, nccf_att_name, att_val);
-         if (!found) found = get_global_att(nc_file, nccf_att_name_l, att_val);
-         if (!found) found = get_global_att(nc_file, nccf_att_name_U, att_val);
+         bool found = get_cf_conventions(nc_file, att_val); // "Conventions" attrribute
          if (found) {
             status = (att_val.compare(0, nccf_att_value.length(),
                       nccf_att_value) == 0  ||

diff --git a/src/libcode/vx_nc_obs/nc_obs_util.cc b/src/libcode/vx_nc_obs/nc_obs_util.cc
@@ -480,6 +480,9 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) {
    obs_dim     = get_nc_dim(f_in, nc_dim_nobs);     // Observation array length
    hdr_dim     = get_nc_dim(f_in, nc_dim_nhdr);     // Header array length
 
+   use_var_id = false;
+   get_global_att(f_in, nc_att_use_var_id, use_var_id);
+
    // Get netCDF header variables
    hdr_typ_var = get_var(f_in, nc_var_hdr_typ);     // Message type (String or int)
    hdr_sid_var = get_var(f_in, nc_var_hdr_sid);     // Station ID (String or int)
@@ -503,10 +506,14 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) {
       obs_arr_var = get_var(f_in, nc_var_obs_arr);
    } else {
       obs_hid_var = ncVar;                             // Obs. header id array
-      ncVar = get_var(f_in, nc_var_obs_gc);
-      if (!IS_INVALID_NC(ncVar)) obs_gc_var  = ncVar;  // Obs. grib code array
-      ncVar = get_var(f_in, nc_var_obs_vid);
-      if (!IS_INVALID_NC(ncVar)) obs_vid_var = ncVar;  // Obs. variable id array
+      if (use_var_id) {
+         ncVar = get_var(f_in, nc_var_obs_vid);
+         if (!IS_INVALID_NC(ncVar)) obs_vid_var = ncVar;  // Obs. variable id array
+      }
+      else {
+         ncVar = get_var(f_in, nc_var_obs_gc);
+         if (!IS_INVALID_NC(ncVar)) obs_gc_var  = ncVar;  // Obs. grib code array
+      }
       obs_lvl_var = get_var(f_in, nc_var_obs_lvl);     // Obs. pressure level array
       obs_hgt_var = get_var(f_in, nc_var_obs_hgt);     // Obs. highth array
       obs_val_var = get_var(f_in, nc_var_obs_val);     // Obs. value array
@@ -516,12 +523,14 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) {
    ncVar = get_var(f_in, nc_var_obs_qty_tbl);
    if (!IS_INVALID_NC(ncVar)) obs_qty_tbl_var = ncVar;
 
-   ncVar = get_var(f_in, nc_var_obs_var);
-   if (!IS_INVALID_NC(ncVar)) obs_var = ncVar;
-   ncVar = get_var(f_in, nc_var_unit);
-   if (!IS_INVALID_NC(ncVar)) unit_var = ncVar;
-   ncVar = get_var(f_in, nc_var_desc);
-   if (!IS_INVALID_NC(ncVar)) desc_var = ncVar;
+   if (use_var_id) {
+      ncVar = get_var(f_in, nc_var_obs_var);
+      if (!IS_INVALID_NC(ncVar)) obs_var = ncVar;
+      ncVar = get_var(f_in, nc_var_unit);
+      if (!IS_INVALID_NC(ncVar)) unit_var = ncVar;
+      ncVar = get_var(f_in, nc_var_desc);
+      if (!IS_INVALID_NC(ncVar)) desc_var = ncVar;
+   }
 
    // PrepBufr only headers
    ncVar = get_var(f_in, nc_var_hdr_prpt_typ);
@@ -531,12 +540,6 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) {
    ncVar = get_var(f_in, nc_var_hdr_inst_typ);
    if (!IS_INVALID_NC(ncVar)) hdr_inst_typ_var = ncVar;
 
-   bool _use_var_id = false;
-   if (!get_global_att(f_in, nc_att_use_var_id, _use_var_id)) {
-      _use_var_id = IS_VALID_NC(obs_var);
-   }
-
-   use_var_id = _use_var_id;
 }
 
 ////////////////////////////////////////////////////////////////////////