From 77a20696f161e05637a4b00bba055b987853e363 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 23 Feb 2021 17:57:03 -0700 Subject: [PATCH 1/6] Per #1135, add fcst/obs_init/valid_inc/exc options for STAT-Analysis jobs. --- met/src/libcode/vx_analysis_util/stat_job.cc | 398 ++++++++++++++----- met/src/libcode/vx_analysis_util/stat_job.h | 8 + 2 files changed, 299 insertions(+), 107 deletions(-) diff --git a/met/src/libcode/vx_analysis_util/stat_job.cc b/met/src/libcode/vx_analysis_util/stat_job.cc index 0f259dc4a1..202bc5068c 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.cc +++ b/met/src/libcode/vx_analysis_util/stat_job.cc @@ -119,15 +119,23 @@ void STATAnalysisJob::clear() { obs_lead.clear(); fcst_valid_beg = fcst_valid_end = (unixtime) 0; + fcst_valid_inc.clear(); + fcst_valid_exc.clear(); fcst_valid_hour.clear(); - obs_valid_beg = obs_valid_end = (unixtime) 0; + obs_valid_beg = obs_valid_end = (unixtime) 0; + obs_valid_inc.clear(); + obs_valid_exc.clear(); obs_valid_hour.clear(); - fcst_init_beg = fcst_init_end = (unixtime) 0; + fcst_init_beg = fcst_init_end = (unixtime) 0; + fcst_init_inc.clear(); + fcst_init_exc.clear(); fcst_init_hour.clear(); - obs_init_beg = obs_init_end = (unixtime) 0; + obs_init_beg = obs_init_end = (unixtime) 0; + obs_init_inc.clear(); + obs_init_exc.clear(); obs_init_hour.clear(); fcst_var.clear(); @@ -242,18 +250,26 @@ void STATAnalysisJob::assign(const STATAnalysisJob & aj) { fcst_valid_beg = aj.fcst_valid_beg; fcst_valid_end = aj.fcst_valid_end; + fcst_valid_inc = aj.fcst_valid_inc; + fcst_valid_exc = aj.fcst_valid_exc; fcst_valid_hour = aj.fcst_valid_hour; obs_valid_beg = aj.obs_valid_beg; obs_valid_end = aj.obs_valid_end; + obs_valid_inc = aj.obs_valid_inc; + obs_valid_exc = aj.obs_valid_exc; obs_valid_hour = aj.obs_valid_hour; fcst_init_beg = aj.fcst_init_beg; fcst_init_end = aj.fcst_init_end; + fcst_init_inc = aj.fcst_init_inc; + fcst_init_exc = aj.fcst_init_exc; fcst_init_hour = aj.fcst_init_hour; obs_init_beg = aj.obs_init_beg; obs_init_end = aj.obs_init_end; + obs_init_inc = aj.obs_init_inc; + obs_init_exc = aj.obs_init_exc; obs_init_hour = aj.obs_init_hour; fcst_var = aj.fcst_var; @@ -370,6 +386,12 @@ void STATAnalysisJob::dump(ostream & out, int depth) const { out << prefix << "fcst_valid_end = " << prefix << timestring(fcst_valid_end) << "\n"; + out << prefix << "fcst_valid_inc ...\n"; + fcst_valid_inc.dump(out, depth + 1); + + out << prefix << "fcst_valid_exc ...\n"; + fcst_valid_exc.dump(out, depth + 1); + out << prefix << "fcst_valid_hour ...\n"; fcst_valid_hour.dump(out, depth + 1); @@ -379,6 +401,12 @@ void STATAnalysisJob::dump(ostream & out, int depth) const { out << prefix << "obs_valid_end = " << prefix << timestring(obs_valid_end) << "\n"; + out << prefix << "obs_valid_inc ...\n"; + obs_valid_inc.dump(out, depth + 1); + + out << prefix << "obs_valid_exc ...\n"; + obs_valid_exc.dump(out, depth + 1); + out << prefix << "obs_valid_hour ...\n"; obs_valid_hour.dump(out, depth + 1); @@ -388,6 +416,12 @@ void STATAnalysisJob::dump(ostream & out, int depth) const { out << prefix << "fcst_init_end = " << prefix << timestring(fcst_init_end) << "\n"; + out << prefix << "fcst_init_inc ...\n"; + fcst_init_inc.dump(out, depth + 1); + + out << prefix << "fcst_init_exc ...\n"; + fcst_init_exc.dump(out, depth + 1); + out << prefix << "fcst_init_hour ...\n"; fcst_init_hour.dump(out, depth + 1); @@ -397,6 +431,12 @@ void STATAnalysisJob::dump(ostream & out, int depth) const { out << prefix << "obs_init_end = " << prefix << timestring(obs_init_end) << "\n"; + out << prefix << "obs_init_inc ...\n"; + obs_init_inc.dump(out, depth + 1); + + out << prefix << "obs_init_exc ...\n"; + obs_init_exc.dump(out, depth + 1); + out << prefix << "obs_init_hour ...\n"; obs_init_hour.dump(out, depth + 1); @@ -592,21 +632,21 @@ int STATAnalysisJob::is_keeper(const STATLine & L) const { // // model // - if(model.n_elements() > 0) { + if(model.n() > 0) { if(!(model.has(L.model()))) return(0); } // // desc // - if(desc.n_elements() > 0) { + if(desc.n() > 0) { if(!(desc.has(L.desc()))) return(0); } // // fcst_lead (in seconds) // - if(fcst_lead.n_elements() > 0) { + if(fcst_lead.n() > 0) { if(!fcst_lead.has(L.fcst_lead())) return(0); } @@ -622,11 +662,27 @@ int STATAnalysisJob::is_keeper(const STATLine & L) const { if((fcst_valid_end > 0) && (L.fcst_valid_end() > fcst_valid_end)) return(0); + // + // fcst_valid_inc + // + if(fcst_valid_inc.n() > 0) { + if(!fcst_valid_inc.has(L.fcst_valid_beg()) || + !fcst_valid_inc.has(L.fcst_valid_end())) return(0); + } + + // + // fcst_valid_exc + // + if(fcst_valid_exc.n() > 0) { + if(fcst_valid_exc.has(L.fcst_valid_beg()) || + fcst_valid_exc.has(L.fcst_valid_end())) return(0); + } + // // fcst_valid_hour: if specified for the job, the fcst_valid_beg and // fcst_valid_end times must match // - if(fcst_valid_hour.n_elements() > 0) { + if(fcst_valid_hour.n() > 0) { // Check that fcst_valid_beg = fcst_valid_end if(L.fcst_valid_beg() != L.fcst_valid_end()) return(0); @@ -646,11 +702,27 @@ int STATAnalysisJob::is_keeper(const STATLine & L) const { if((fcst_init_end > 0) && (L.fcst_init_end() > fcst_init_end)) return(0); + // + // fcst_init_inc + // + if(fcst_init_inc.n() > 0) { + if(!fcst_init_inc.has(L.fcst_init_beg()) || + !fcst_init_inc.has(L.fcst_init_end())) return(0); + } + + // + // fcst_init_exc + // + if(fcst_init_exc.n() > 0) { + if(fcst_init_exc.has(L.fcst_init_beg()) || + fcst_init_exc.has(L.fcst_init_end())) return(0); + } + // // fcst_init_hour: if specified for the job, the fcst_init_beg and // fcst_init_end times must match // - if(fcst_init_hour.n_elements() > 0) { + if(fcst_init_hour.n() > 0) { // Check that fcst_init_beg = fcst_init_end if(L.fcst_init_beg() != L.fcst_init_end()) return(0); @@ -661,7 +733,7 @@ int STATAnalysisJob::is_keeper(const STATLine & L) const { // // obs_lead (in seconds) // - if(obs_lead.n_elements() > 0) { + if(obs_lead.n() > 0) { if(!obs_lead.has(L.obs_lead())) return(0); } @@ -677,11 +749,27 @@ int STATAnalysisJob::is_keeper(const STATLine & L) const { if((obs_valid_end > 0) && (L.obs_valid_end() > obs_valid_end)) return(0); + // + // obs_valid_inc + // + if(obs_valid_inc.n() > 0) { + if(!obs_valid_inc.has(L.obs_valid_beg()) || + !obs_valid_inc.has(L.obs_valid_end())) return(0); + } + + // + // obs_valid_exc + // + if(obs_valid_exc.n() > 0) { + if(obs_valid_exc.has(L.obs_valid_beg()) || + obs_valid_exc.has(L.obs_valid_end())) return(0); + } + // // obs_valid_hour: if specified for the job, the obs_valid_beg and // obs_valid_end times must match // - if(obs_valid_hour.n_elements() > 0) { + if(obs_valid_hour.n() > 0) { // Check that obs_valid_beg = obs_valid_end if(L.obs_valid_beg() != L.obs_valid_end()) return(0); @@ -701,11 +789,27 @@ int STATAnalysisJob::is_keeper(const STATLine & L) const { if((obs_init_end > 0) && (L.obs_init_end() > obs_init_end)) return(0); + // + // obs_init_inc + // + if(obs_init_inc.n() > 0) { + if(!obs_init_inc.has(L.obs_init_beg()) || + !obs_init_inc.has(L.obs_init_end())) return(0); + } + + // + // obs_init_exc + // + if(obs_init_exc.n() > 0) { + if(obs_init_exc.has(L.obs_init_beg()) || + obs_init_exc.has(L.obs_init_end())) return(0); + } + // // obs_init_hour: if specified for the job, the obs_init_beg and // obs_init_end times must match // - if(obs_init_hour.n_elements() > 0) { + if(obs_init_hour.n() > 0) { // Check that obs_init_beg = obs_init_end if(L.obs_init_beg() != L.obs_init_end()) return(0); @@ -716,91 +820,91 @@ int STATAnalysisJob::is_keeper(const STATLine & L) const { // // fcst_var // - if(fcst_var.n_elements() > 0) { + if(fcst_var.n() > 0) { if(!(fcst_var.has(L.fcst_var()))) return(0); } // // fcst_units // - if(fcst_units.n_elements() > 0) { + if(fcst_units.n() > 0) { if(!(fcst_units.has(L.fcst_units()))) return(0); } // // fcst_lev // - if(fcst_lev.n_elements() > 0) { + if(fcst_lev.n() > 0) { if(!(fcst_lev.has(L.fcst_lev()))) return(0); } // // obs_var // - if(obs_var.n_elements() > 0) { + if(obs_var.n() > 0) { if(!(obs_var.has(L.obs_var()))) return(0); } // // obs_units // - if(obs_units.n_elements() > 0) { + if(obs_units.n() > 0) { if(!(obs_units.has(L.obs_units()))) return(0); } // // obs_lev // - if(obs_lev.n_elements() > 0) { + if(obs_lev.n() > 0) { if(!(obs_lev.has(L.obs_lev()))) return(0); } // // obtype // - if(obtype.n_elements() > 0) { + if(obtype.n() > 0) { if(!(obtype.has(L.obtype()))) return(0); } // // vx_mask // - if(vx_mask.n_elements() > 0) { + if(vx_mask.n() > 0) { if(!(vx_mask.has(L.vx_mask()))) return(0); } // // interp_mthd // - if(interp_mthd.n_elements() > 0) { + if(interp_mthd.n() > 0) { if(!(interp_mthd.has(L.interp_mthd()))) return(0); } // // interp_pnts // - if(interp_pnts.n_elements() > 0) { + if(interp_pnts.n() > 0) { if(!interp_pnts.has(L.interp_pnts())) return(0); } // // fcst_thresh // - if(fcst_thresh.n_elements() > 0) { + if(fcst_thresh.n() > 0) { if(!check_thresh_column(fcst_thresh, L.fcst_thresh())) return(0); } // // obs_thresh // - if(obs_thresh.n_elements() > 0) { + if(obs_thresh.n() > 0) { if(!check_thresh_column(obs_thresh, L.obs_thresh())) return(0); } // // cov_thresh // - if(cov_thresh.n_elements() > 0) { + if(cov_thresh.n() > 0) { if(!check_thresh_column(cov_thresh, L.cov_thresh())) return(0); } @@ -813,14 +917,14 @@ int STATAnalysisJob::is_keeper(const STATLine & L) const { // // alpha // - if(alpha.n_elements() > 0) { + if(alpha.n() > 0) { if(!alpha.has(L.alpha())) return(0); } // // line_type // - if(line_type.n_elements() > 0) { + if(line_type.n() > 0) { if(!(line_type.has(L.line_type()))) return(0); } @@ -895,10 +999,10 @@ double STATAnalysisJob::get_column_double(const STATLine &L, v = atof(L.get_item(sa[0].c_str())); // If multiple columns, compute the requested difference - if(sa.n_elements() > 1) { + if(sa.n() > 1) { // Loop through the column - for(i=1; i 0 ? + out_sa = (out_line_type.n() > 0 ? out_line_type : line_type); - out_lt = (out_sa.n_elements() == 1 ? + out_lt = (out_sa.n() == 1 ? string_to_statlinetype(out_sa[0].c_str()) : no_stat_line_type); // // Loop through the output line types and determine the number of // output columns // - for(i=0, c=0, n_col=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { + for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { + for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { + for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { + for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i::const_iterator thr_it = column_thresh_map.begin(); thr_it != column_thresh_map.end(); thr_it++) { - for(i=0; isecond.n_elements(); i++) { + for(i=0; isecond.n(); i++) { js << "-column_thresh " << thr_it->first << " " << thr_it->second[i].get_str() << " "; } } @@ -2280,20 +2464,20 @@ ConcatString STATAnalysisJob::get_jobstring() const { for(map::const_iterator str_it = column_str_map.begin(); str_it != column_str_map.end(); str_it++) { - for(i=0; isecond.n_elements(); i++) { + for(i=0; isecond.n(); i++) { js << "-column_str " << str_it->first << " " << str_it->second[i] << " "; } } // by_column - if(by_column.n_elements() > 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0) { - for(i=0; i 0) { + for(i=0; i 0 || out_obs_thresh.n_elements() > 0)) { + (out_fcst_thresh.n() > 0 || out_obs_thresh.n() > 0)) { js << "-out_cnt_logic " << setlogic_to_string(out_cnt_logic) << " "; } @@ -2437,7 +2621,7 @@ ConcatString STATAnalysisJob::get_jobstring() const { } // Jobs which use out_bin_size - if(line_type.n_elements() > 0) { + if(line_type.n() > 0) { if(string_to_statlinetype(line_type[0].c_str()) == stat_orank && out_line_type.has(stat_phist_str)) { @@ -2447,19 +2631,19 @@ ConcatString STATAnalysisJob::get_jobstring() const { } // Jobs which use out_eclv_points - if(line_type.n_elements() > 0) { + if(line_type.n() > 0) { if(string_to_statlinetype(line_type[0].c_str()) == stat_mpr && out_line_type.has(stat_eclv_str)) { // out_eclv_points - for(i=0; i 0) { + if(line_type.n() > 0) { type = string_to_statlinetype(line_type[0].c_str()); if(type == stat_mpr && (out_line_type.has(stat_cts_str) || @@ -2552,7 +2736,7 @@ int STATAnalysisJob::is_in_mask_sid(const char *sid) const { // // Only check if a masking SID list has been specified // - if(mask_sid.n_elements() > 0) r = mask_sid.has(sid); + if(mask_sid.n() > 0) r = mask_sid.has(sid); return(r); } @@ -2621,10 +2805,10 @@ ConcatString timestring(const unixtime t) { bool check_thresh_column(const ThreshArray &list, const ThreshArray &item) { // Return true for an empty search list. - if(list.n_elements() == 0) return(true); + if(list.n() == 0) return(true); // If the item is a single threshold, search for it in the list. - if(item.n_elements() == 1) { + if(item.n() == 1) { return(list.has(item[0])); } diff --git a/met/src/libcode/vx_analysis_util/stat_job.h b/met/src/libcode/vx_analysis_util/stat_job.h index 1fde44b9cd..c86a6d84aa 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.h +++ b/met/src/libcode/vx_analysis_util/stat_job.h @@ -172,18 +172,26 @@ class STATAnalysisJob { unixtime fcst_valid_beg; unixtime fcst_valid_end; + TimeArray fcst_valid_inc; + TimeArray fcst_valid_exc; IntArray fcst_valid_hour; // stored in seconds unixtime obs_valid_beg; unixtime obs_valid_end; + TimeArray obs_valid_inc; + TimeArray obs_valid_exc; IntArray obs_valid_hour; // stored in seconds unixtime fcst_init_beg; unixtime fcst_init_end; + TimeArray fcst_init_inc; + TimeArray fcst_init_exc; IntArray fcst_init_hour; // stored in seconds unixtime obs_init_beg; unixtime obs_init_end; + TimeArray obs_init_inc; + TimeArray obs_init_exc; IntArray obs_init_hour; // stored in seconds StringArray fcst_var; From 4f729816d0cb3c53f99b1bd114c9d191c65a56da Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 23 Feb 2021 17:57:56 -0700 Subject: [PATCH 2/6] Per #1135, update all the STATAnalysis config files to include entries for the new fcst/obs_init/valid_inc/exc options. --- met/data/config/STATAnalysisConfig_GO_Index | 8 ++++++++ met/data/config/STATAnalysisConfig_default | 8 ++++++++ met/scripts/config/STATAnalysisConfig | 8 ++++++++ test/config/STATAnalysisConfig_APCP_HIRA | 8 ++++++++ test/config/STATAnalysisConfig_climo | 8 ++++++++ test/config/STATAnalysisConfig_grid_stat | 8 ++++++++ test/config/STATAnalysisConfig_point_stat | 8 ++++++++ test/config/STATAnalysisConfig_ramps | 8 ++++++++ 8 files changed, 64 insertions(+) diff --git a/met/data/config/STATAnalysisConfig_GO_Index b/met/data/config/STATAnalysisConfig_GO_Index index 121ec12516..4f854fdf30 100644 --- a/met/data/config/STATAnalysisConfig_GO_Index +++ b/met/data/config/STATAnalysisConfig_GO_Index @@ -28,18 +28,26 @@ obs_lead = []; fcst_valid_beg = ""; fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; fcst_valid_hour = []; obs_valid_beg = ""; obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; obs_valid_hour = []; fcst_init_beg = ""; fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; fcst_init_hour = []; obs_init_beg = ""; obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; obs_init_hour = []; fcst_var = [ "WIND", "WIND", "WIND", "WIND", diff --git a/met/data/config/STATAnalysisConfig_default b/met/data/config/STATAnalysisConfig_default index d0f507663a..bc4a50ca44 100644 --- a/met/data/config/STATAnalysisConfig_default +++ b/met/data/config/STATAnalysisConfig_default @@ -17,18 +17,26 @@ obs_lead = []; fcst_valid_beg = ""; fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; fcst_valid_hour = []; obs_valid_beg = ""; obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; obs_valid_hour = []; fcst_init_beg = ""; fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; fcst_init_hour = []; obs_init_beg = ""; obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; obs_init_hour = []; fcst_var = []; diff --git a/met/scripts/config/STATAnalysisConfig b/met/scripts/config/STATAnalysisConfig index 3518016e21..4189ff537d 100644 --- a/met/scripts/config/STATAnalysisConfig +++ b/met/scripts/config/STATAnalysisConfig @@ -17,18 +17,26 @@ obs_lead = []; fcst_valid_beg = ""; fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; fcst_valid_hour = []; obs_valid_beg = ""; obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; obs_valid_hour = []; fcst_init_beg = ""; fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; fcst_init_hour = []; obs_init_beg = ""; obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; obs_init_hour = []; fcst_var = []; diff --git a/test/config/STATAnalysisConfig_APCP_HIRA b/test/config/STATAnalysisConfig_APCP_HIRA index 228f6f16fb..b1ca1898d0 100644 --- a/test/config/STATAnalysisConfig_APCP_HIRA +++ b/test/config/STATAnalysisConfig_APCP_HIRA @@ -17,18 +17,26 @@ obs_lead = []; fcst_valid_beg = ""; fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; fcst_valid_hour = []; obs_valid_beg = ""; obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; obs_valid_hour = []; fcst_init_beg = ""; fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; fcst_init_hour = []; obs_init_beg = ""; obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; obs_init_hour = []; fcst_var = [ "APCP_24" ]; diff --git a/test/config/STATAnalysisConfig_climo b/test/config/STATAnalysisConfig_climo index 165ac89b42..969fbac88e 100644 --- a/test/config/STATAnalysisConfig_climo +++ b/test/config/STATAnalysisConfig_climo @@ -17,18 +17,26 @@ obs_lead = []; fcst_valid_beg = ""; fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; fcst_valid_hour = []; obs_valid_beg = ""; obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; obs_valid_hour = []; fcst_init_beg = ""; fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; fcst_init_hour = []; obs_init_beg = ""; obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; obs_init_hour = []; fcst_var = []; diff --git a/test/config/STATAnalysisConfig_grid_stat b/test/config/STATAnalysisConfig_grid_stat index 8a2725282a..b29fcbe58a 100644 --- a/test/config/STATAnalysisConfig_grid_stat +++ b/test/config/STATAnalysisConfig_grid_stat @@ -17,18 +17,26 @@ obs_lead = []; fcst_valid_beg = ""; fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; fcst_valid_hour = []; obs_valid_beg = ""; obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; obs_valid_hour = []; fcst_init_beg = ""; fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; fcst_init_hour = []; obs_init_beg = ""; obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; obs_init_hour = []; fcst_var = [ "TMP" ]; diff --git a/test/config/STATAnalysisConfig_point_stat b/test/config/STATAnalysisConfig_point_stat index 03fbf0e21f..68e1d17901 100644 --- a/test/config/STATAnalysisConfig_point_stat +++ b/test/config/STATAnalysisConfig_point_stat @@ -17,18 +17,26 @@ obs_lead = []; fcst_valid_beg = ""; fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; fcst_valid_hour = []; obs_valid_beg = ""; obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; obs_valid_hour = []; fcst_init_beg = ""; fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; fcst_init_hour = []; obs_init_beg = ""; obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; obs_init_hour = []; fcst_var = []; diff --git a/test/config/STATAnalysisConfig_ramps b/test/config/STATAnalysisConfig_ramps index ab6faa1051..cd6ec9f268 100644 --- a/test/config/STATAnalysisConfig_ramps +++ b/test/config/STATAnalysisConfig_ramps @@ -17,18 +17,26 @@ obs_lead = []; fcst_valid_beg = ""; fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; fcst_valid_hour = []; obs_valid_beg = ""; obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; obs_valid_hour = []; fcst_init_beg = ""; fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; fcst_init_hour = []; obs_init_beg = ""; obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; obs_init_hour = []; fcst_var = []; From 6c859abd8c2c58de2f8766597368b3f9b85ba74b Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 23 Feb 2021 17:59:01 -0700 Subject: [PATCH 3/6] Per #1135, add documentation for fcst/obs_init/valid_inc/exc options to the STAT-Analysis chapter. Also, clarify the description for the existing options. --- met/docs/Users_Guide/stat-analysis.rst | 28 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index 107cc3c279..d9c75ad113 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -303,12 +303,21 @@ ___________________ fcst_valid_beg = ""; fcst_valid_end = ""; - fcst_valid_hour = ""; + fcst_valid_inc = []; + fcst_valid_exc = []; + fcst_valid_hour = []; + obs_valid_beg = ""; - obs_valid_end = "" - obs_valid_hour = ""; + obs_valid_end = ""; + obs_valid_inc = []; + obs_valid_exc = []; + obs_valid_hour = []; + +The user may filter data based on its valid time. The fcst/obs_valid_beg and fcst/obs_valid_end options are strings in YYYYMMDD[_HH[MMSS]] format which define retention time windows for all analyses to be performed. The analyses are performed on all data whose valid time falls within these windows. If left as empty strings, no valid time window filtering is applied. -The user may specify the beginning, ending, and instantaneous valid times in YYYYMMDD[_HH[MMSS]] format to be used for all analyses performed. If multiple valid times fall within the valid time window, the analyses will be performed on their union. These selections may be further refined by using the **"-fcst_valid_beg", "-fcst_valid_end", "-obs_valid_beg", "-obs_valid_end", “fcst_valid_hour"** and **"-obs_valid_hour"** options within the job command line. +The fcst/obs_valid_hour options are arrays of strings in HH format which define the valid hour(s) of the data to be used. If specified, only data whose valid hour appears in the list of hours is used. The fcst/obs_valid_inc/exc options are arrays of strings in YYYYMMDD[_HH[MMSS]] format which explicitly define the valid times for data to be included or excluded from all analyses. + +These selections may be further refined by using the **"-fcst_valid_beg", "-fcst_valid_end", "-fcst_valid_inc", "-fcst_valid_exc", "-fcst_valid_hour", "-obs_valid_beg", "-obs_valid_end", "-obs_valid_inc", "-obs_valid_exc",** and **"-obs_valid_hour"** options within the job command line. ___________________ @@ -316,12 +325,17 @@ ___________________ fcst_init_beg = ""; fcst_init_end = ""; - fcst_init_hour = ""; + fcst_init_inc = []; + fcst_init_exc = []; + fcst_init_hour = []; + obs_init_beg = ""; obs_init_end = ""; - obs_init_hour = ""; + obs_init_inc = []; + obs_init_exc = []; + obs_init_hour = []; -The user may specify the beginning, ending, or exact model initialization times in YYYYMMDD[_HH[MMSS]] format to be used for all analyses performed. If multiple init times fall within the init time window, the analyses will be performed on their union. These selections may be further refined by using the **"-fcst_init_beg", "-fcst_init_end", "-obs_init_beg", "-obs_init_end", fcst_init_hour"** and **"-obs_init_hour"** options within the job command line. +These time filtering options are the same as described above but applied to initialization times rather than valid times. These selections may be further refined by using the **"-fcst_init_beg", "-fcst_init_end", "-fcst_init_inc", "-fcst_init_exc", "-fcst_init_hour"," "-obs_init_beg", "-obs_init_end", fcst_init_hour"** and **"-obs_init_hour"** options within the job command line. ___________________ From fd57e93d7f5c7584fcfc0e96387971318e62f393 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 24 Feb 2021 11:40:54 -0700 Subject: [PATCH 4/6] Per #1135, adding another call to stat_analysis to check the time filtering options. --- test/config/STATAnalysisConfig_filter_times | 109 ++++++++++++++++++++ test/xml/unit_stat_analysis.xml | 16 +++ 2 files changed, 125 insertions(+) create mode 100644 test/config/STATAnalysisConfig_filter_times diff --git a/test/config/STATAnalysisConfig_filter_times b/test/config/STATAnalysisConfig_filter_times new file mode 100644 index 0000000000..b9f296d24b --- /dev/null +++ b/test/config/STATAnalysisConfig_filter_times @@ -0,0 +1,109 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// STAT-Analysis configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Filtering input STAT lines by the contents of each column +// +model = []; +desc = []; + +fcst_lead = []; +obs_lead = []; + +fcst_valid_beg = ""; +fcst_valid_end = ""; +fcst_valid_inc = [ "20120409_12" ]; +fcst_valid_exc = []; +fcst_valid_hour = [ "12" ]; + +obs_valid_beg = ""; +obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; +obs_valid_hour = []; + +fcst_init_beg = ""; +fcst_init_end = ""; +fcst_init_inc = [ "20120409_00" ]; +fcst_init_exc = []; +fcst_init_hour = [ "00" ]; + +obs_init_beg = ""; +obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; +obs_init_hour = []; + +fcst_var = [ "TMP" ]; +obs_var = []; + +fcst_lev = [ "Z2" ]; +obs_lev = []; + +obtype = []; + +vx_mask = [ "DTC165" ]; + +interp_mthd = [ "NEAREST" ]; + +interp_pnts = [ 1 ]; + +fcst_thresh = []; +obs_thresh = []; +cov_thresh = []; + +alpha = []; + +line_type = [ "MPR" ]; + +column = []; + +weight = []; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Array of STAT-Analysis jobs to be performed on the filtered data +// +// Job 1 = all points +// Job 2 + Job 3 = all Job 1 points +// Job 4 = subset of all points +// Job 5 and Job 6 should match no points and print a warning message +// +jobs = [ + "-job aggregate_stat -out_line_type SL1L2", + "-job aggregate_stat -out_line_type SL1L2 -obs_valid_inc 20120409_12", + "-job aggregate_stat -out_line_type SL1L2 -obs_valid_exc 20120409_12", + "-job aggregate_stat -out_line_type SL1L2 -obs_valid_beg 20120409_115500 -obs_valid_end 20120409_120500", + "-job aggregate_stat -out_line_type SL1L2 -fcst_valid_exc 20120409_12", + "-job aggregate_stat -out_line_type SL1L2 -fcst_init_exc 20120409_00" +]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +out_alpha = 0.05; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 1000; + rng = "mt19937"; + seed = "1"; +} + +//////////////////////////////////////////////////////////////////////////////// + +rank_corr_flag = TRUE; +vif_flag = FALSE; +tmp_dir = "/tmp"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/test/xml/unit_stat_analysis.xml b/test/xml/unit_stat_analysis.xml index 5bb29c5283..f8c91fff2b 100644 --- a/test/xml/unit_stat_analysis.xml +++ b/test/xml/unit_stat_analysis.xml @@ -299,4 +299,20 @@ + + + OUTPUT_DIR &OUTPUT_DIR;/stat_analysis + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB2_NAM_NDAS_120000L_20120409_120000V.stat \ + -config &CONFIG_DIR;/STATAnalysisConfig_filter_times \ + -out &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_TIMES_sl1l2.out \ + -v 1 + + + &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_TIMES_sl1l2.out + + + From 3439185180d8942e290bf8a01c5e18532efa34bb Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 24 Feb 2021 11:49:27 -0700 Subject: [PATCH 5/6] Per #1135, just renaming stat_analysis output file. --- test/xml/unit_stat_analysis.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/xml/unit_stat_analysis.xml b/test/xml/unit_stat_analysis.xml index f8c91fff2b..0d5fa6edc4 100644 --- a/test/xml/unit_stat_analysis.xml +++ b/test/xml/unit_stat_analysis.xml @@ -307,11 +307,11 @@ \ -lookin &OUTPUT_DIR;/point_stat/point_stat_GRIB2_NAM_NDAS_120000L_20120409_120000V.stat \ -config &CONFIG_DIR;/STATAnalysisConfig_filter_times \ - -out &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_TIMES_sl1l2.out \ + -out &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_TIMES.out \ -v 1 - &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_TIMES_sl1l2.out + &OUTPUT_DIR;/stat_analysis/POINT_STAT_FILTER_TIMES.out From 632f10ef72af0e911d60a3c60c3cc051ae5a3046 Mon Sep 17 00:00:00 2001 From: johnhg Date: Wed, 24 Feb 2021 15:42:00 -0700 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: jprestop --- met/docs/Users_Guide/stat-analysis.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index d9c75ad113..50655fc573 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -335,7 +335,7 @@ ___________________ obs_init_exc = []; obs_init_hour = []; -These time filtering options are the same as described above but applied to initialization times rather than valid times. These selections may be further refined by using the **"-fcst_init_beg", "-fcst_init_end", "-fcst_init_inc", "-fcst_init_exc", "-fcst_init_hour"," "-obs_init_beg", "-obs_init_end", fcst_init_hour"** and **"-obs_init_hour"** options within the job command line. +These time filtering options are the same as described above but applied to initialization times rather than valid times. These selections may be further refined by using the **"-fcst_init_beg", "-fcst_init_end", "-fcst_init_inc", "-fcst_init_exc", "-fcst_init_hour"," "-obs_init_beg", "-obs_init_end", "-obs_init_inc", "-obs_init_exc"** and **"-obs_init_hour"** options within the job command line. ___________________