From af43755f6d6bb6f5a8527621a98c0697a64010a0 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 16 Jul 2021 16:12:00 -0600 Subject: [PATCH 01/60] Per #1788, add initial definition of the SSIDX line type. --- .../table_files/met_header_columns_V10.1.txt | 1 + met/src/basic/vx_config/config_constants.h | 2 + met/src/libcode/vx_stat_out/stat_columns.cc | 57 +++++++++++++++++++ met/src/libcode/vx_stat_out/stat_columns.h | 4 ++ test/hdr/met_10_1.hdr | 1 + 5 files changed, 65 insertions(+) diff --git a/met/data/table_files/met_header_columns_V10.1.txt b/met/data/table_files/met_header_columns_V10.1.txt index 6c64d89a5f..89e483e7e7 100644 --- a/met/data/table_files/met_header_columns_V10.1.txt +++ b/met/data/table_files/met_header_columns_V10.1.txt @@ -29,6 +29,7 @@ V10.1 : STAT : VAL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID V10.1 : STAT : VL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL UFBAR VFBAR UOBAR VOBAR UVFOBAR UVFFBAR UVOOBAR F_SPEED_BAR O_SPEED_BAR V10.1 : STAT : VCNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FBAR FBAR_BCL FBAR_BCU OBAR OBAR_BCL OBAR_BCU FS_RMS FS_RMS_BCL FS_RMS_BCU OS_RMS OS_RMS_BCL OS_RMS_BCU MSVE MSVE_BCL MSVE_BCU RMSVE RMSVE_BCL RMSVE_BCU FSTDEV FSTDEV_BCL FSTDEV_BCU OSTDEV OSTDEV_BCL OSTDEV_BCU FDIR FDIR_BCL FDIR_BCU ODIR ODIR_BCL ODIR_BCU FBAR_SPEED FBAR_SPEED_BCL FBAR_SPEED_BCU OBAR_SPEED OBAR_SPEED_BCL OBAR_SPEED_BCU VDIFF_SPEED VDIFF_SPEED_BCL VDIFF_SPEED_BCU VDIFF_DIR VDIFF_DIR_BCL VDIFF_DIR_BCU SPEED_ERR SPEED_ERR_BCL SPEED_ERR_BCU SPEED_ABSERR SPEED_ABSERR_BCL SPEED_ABSERR_BCU DIR_ERR DIR_ERR_BCL DIR_ERR_BCU DIR_ABSERR DIR_ABSERR_BCL DIR_ABSERR_BCU V10.1 : STAT : GENMPR : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX STORM_ID AGEN_INIT AGEN_FHR AGEN_LAT AGEN_LON AGEN_DLAND BGEN_LAT BGEN_LON BGEN_DLAND GEN_DIST GEN_TDIFF INIT_TDIFF DEV_CAT OPS_CAT +V10.1 : STAT : SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE SS_INDEX V10.1 : MODE : OBJ : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE OBJECT_ID OBJECT_CAT CENTROID_X CENTROID_Y CENTROID_LAT CENTROID_LON AXIS_ANG LENGTH WIDTH AREA AREA_THRESH CURVATURE CURVATURE_X CURVATURE_Y COMPLEXITY INTENSITY_10 INTENSITY_25 INTENSITY_50 INTENSITY_75 INTENSITY_90 INTENSITY_USER INTENSITY_SUM CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST V10.1 : MODE : CTS : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index d3b8773e58..ce37cbf89f 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -137,6 +137,7 @@ enum STATLineType { stat_grad, stat_dmap, stat_genmpr, + stat_ssidx, stat_header, no_stat_line_type @@ -181,6 +182,7 @@ static const char stat_orank_str[] = "ORANK"; static const char stat_ssvar_str[] = "SSVAR"; static const char stat_relp_str[] = "RELP"; static const char stat_genmpr_str[] = "GENMPR"; +static const char stat_ssidx_str[] = "SSIDX"; static const char stat_header_str[] = "LINE_TYPE"; static const char stat_na_str[] = "NA"; diff --git a/met/src/libcode/vx_stat_out/stat_columns.cc b/met/src/libcode/vx_stat_out/stat_columns.cc index e385cfcfc7..fce5a3ac38 100644 --- a/met/src/libcode/vx_stat_out/stat_columns.cc +++ b/met/src/libcode/vx_stat_out/stat_columns.cc @@ -1892,6 +1892,47 @@ void write_relp_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr, //////////////////////////////////////////////////////////////////////// +void write_ssidx_row(StatHdrColumns &shc, double ss_index, + STATOutputType out_type, + AsciiTable &stat_at, int &stat_row, + AsciiTable &txt_at, int &txt_row) { + int i; + + // SSIDX line type + shc.set_line_type(stat_ssidx_str); + + // Not Applicable + shc.set_interp_mthd(InterpMthd_None, + GridTemplateFactory::GridTemplate_None); + shc.set_interp_wdth(bad_data_int); + shc.set_fcst_thresh(na_str); + shc.set_obs_thresh(na_str); + shc.set_thresh_logic(SetLogic_None); + shc.set_cov_thresh(na_str); + shc.set_alpha(bad_data_double); + + // Write the header columns + write_header_cols(shc, stat_at, stat_row); + + // Write the data columns + write_ssidx_cols(ss_index, stat_at, stat_row, n_header_columns); + + // If requested, copy row to the text file + if(out_type == STATOutputType_Both) { + copy_ascii_table_row(stat_at, stat_row, txt_at, txt_row); + + // Increment the text row counter + txt_row++; + } + + // Increment the STAT row counter + stat_row++; + + return; +} + +//////////////////////////////////////////////////////////////////////// + void write_header_cols(const StatHdrColumns &shc, AsciiTable &at, int r) { ConcatString cs; @@ -4106,6 +4147,22 @@ void write_relp_cols(const PairDataEnsemble *pd_ptr, //////////////////////////////////////////////////////////////////////// +void write_ssidx_cols(double ss_index, + AsciiTable &at, int r, int c) { + + // + // Skill Score Index + // Dump out the SSIDX line: + // SS_INDEX + // + at.set_entry(r, c+0, // Skill Score Index + ss_index); + + return; +} + +//////////////////////////////////////////////////////////////////////// + void justify_stat_cols(AsciiTable &at) { justify_met_at(at, n_header_columns); diff --git a/met/src/libcode/vx_stat_out/stat_columns.h b/met/src/libcode/vx_stat_out/stat_columns.h index ff974d7c1e..762c959464 100644 --- a/met/src/libcode/vx_stat_out/stat_columns.h +++ b/met/src/libcode/vx_stat_out/stat_columns.h @@ -120,6 +120,8 @@ extern void write_ssvar_row (StatHdrColumns &, const PairDataEnsemble *, double, AsciiTable &, int &, AsciiTable &, int &); extern void write_relp_row (StatHdrColumns &, const PairDataEnsemble *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); +extern void write_ssidx_row (StatHdrColumns &, double, STATOutputType, + AsciiTable &, int &, AsciiTable &, int &); //////////////////////////////////////////////////////////////////////// @@ -189,6 +191,8 @@ extern void write_ssvar_cols (const PairDataEnsemble *, int, double, AsciiTable &, int, int); extern void write_relp_cols (const PairDataEnsemble *, AsciiTable &, int, int); +extern void write_ssidx_cols (double, + AsciiTable &, int, int); // Setup column justification for STAT AsciiTable objects extern void justify_stat_cols(AsciiTable &); diff --git a/test/hdr/met_10_1.hdr b/test/hdr/met_10_1.hdr index 77d4fa0053..8355756271 100644 --- a/test/hdr/met_10_1.hdr +++ b/test/hdr/met_10_1.hdr @@ -29,6 +29,7 @@ VL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L VAL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL UFABAR VFABAR UOABAR VOABAR UVFOABAR UVFFABAR UVOOABAR VCNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FBAR FBAR_BCL FBAR_BCU OBAR OBAR_BCL OBAR_BCU FS_RMS FS_RMS_BCL FS_RMS_BCU OS_RMS OS_RMS_BCL OS_RMS_BCU MSVE MSVE_BCL MSVE_BCU RMSVE RMSVE_BCL RMSVE_BCU FSTDEV FSTDEV_BCL FSTDEV_BCU OSTDEV OSTDEV_BCL OSTDEV_BCU FDIR FDIR_BCL FDIR_BCU ODIR ODIR_BCL ODIR_BCU FBAR_SPEED FBAR_SPEED_BCL FBAR_SPEED_BCU OBAR_SPEED OBAR_SPEED_BCL OBAR_SPEED_BCU VDIFF_SPEED VDIFF_SPEED_BCL VDIFF_SPEED_BCU VDIFF_DIR VDIFF_DIR_BCL VDIFF_DIR_BCU SPEED_ERR SPEED_ERR_BCL SPEED_ERR_BCU SPEED_ABSERR SPEED_ABSERR_BCL SPEED_ABSERR_BCU DIR_ERR DIR_ERR_BCL DIR_ERR_BCU DIR_ABSERR DIR_ABSERR_BCL DIR_ABSERR_BCU GENMPR : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX STORM_ID AGEN_INIT AGEN_FHR AGEN_LAT AGEN_LON AGEN_DLAND BGEN_LAT BGEN_LON BGEN_DLAND GEN_DIST GEN_TDIFF INIT_TDIFF DEV_CAT OPS_CAT +SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE SS_INDEX MODE_SOA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES OBJECT_ID OBJECT_CAT CENTROID_X CENTROID_Y CENTROID_LAT CENTROID_LON AXIS_ANG LENGTH WIDTH AREA AREA_THRESH CURVATURE CURVATURE_X CURVATURE_Y COMPLEXITY INTENSITY_10 INTENSITY_25 INTENSITY_50 INTENSITY_75 INTENSITY_90 INTENSITY_50 INTENSITY_SUM MODE_POA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES OBJECT_ID OBJECT_CAT CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST MODE_CTS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS From 97f8c9e5d075d14190fc5a7d2d1df89b70c4c082 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 16 Jul 2021 16:26:36 -0600 Subject: [PATCH 02/60] Per #1788, add a STAT-Analysis config file for computing the CBS Score. This is currently just a copy of the GO Index. --- met/data/config/Makefile.am | 1 + met/data/config/STATAnalysisConfig_CBS_Score | 162 +++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 met/data/config/STATAnalysisConfig_CBS_Score diff --git a/met/data/config/Makefile.am b/met/data/config/Makefile.am index fe9b468a48..0ab2bd0927 100644 --- a/met/data/config/Makefile.am +++ b/met/data/config/Makefile.am @@ -35,6 +35,7 @@ config_DATA = \ SeriesAnalysisConfig_default \ STATAnalysisConfig_default \ STATAnalysisConfig_GO_Index \ + STATAnalysisConfig_CBS_Score \ TCPairsConfig_default \ TCRMWConfig_default \ RMWAnalysisConfig_default \ diff --git a/met/data/config/STATAnalysisConfig_CBS_Score b/met/data/config/STATAnalysisConfig_CBS_Score new file mode 100644 index 0000000000..8775d5ae37 --- /dev/null +++ b/met/data/config/STATAnalysisConfig_CBS_Score @@ -0,0 +1,162 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// STAT-Analysis configuration file used to compute the CBS Score. +// +// WARNING: This file should not be modified. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Filtering input STAT lines by the contents of each column +// +model = []; + +fcst_lead = [ "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48" + ]; +obs_lead = []; + +fcst_valid_beg = ""; +fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; +fcst_valid_hour = []; + +obs_valid_beg = ""; +obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; +obs_valid_hour = []; + +fcst_init_beg = ""; +fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; +fcst_init_hour = []; + +obs_init_beg = ""; +obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; +obs_init_hour = []; + +fcst_var = [ "WIND", "WIND", "WIND", "WIND", + "WIND", "WIND", "WIND", "WIND", + "WIND", "WIND", "WIND", "WIND", + "WIND", "WIND", "WIND", "WIND", + "DPT", "DPT", "DPT", "DPT", + "DPT", "DPT", "DPT", "DPT", + "DPT", "DPT", "DPT", "DPT", + "DPT", "DPT", "DPT", "DPT", + "TMP", "TMP", "TMP", "TMP", + "TMP", "TMP", "TMP", "TMP", + "HGT", "HGT", "HGT", "HGT", + "PRMSL", "PRMSL", "PRMSL", "PRMSL" + ]; +obs_var = []; + +fcst_lev = [ "P250", "P250", "P250", "P250", + "P400", "P400", "P400", "P400", + "P850", "P850", "P850", "P850", + "Z10", "Z10", "Z10", "Z10", + "P400", "P400", "P400", "P400", + "P700", "P700", "P700", "P700", + "P850", "P850", "P850", "P850", + "Z2", "Z2", "Z2", "Z2", + "P400", "P400", "P400", "P400", + "Z2", "Z2", "Z2", "Z2", + "P400", "P400", "P400", "P400", + "Z0", "Z0", "Z0", "Z0" + ]; +obs_lev = []; + +obtype = []; + +vx_mask = []; + +interp_mthd = []; + +interp_pnts = []; + +fcst_thresh = []; +obs_thresh = []; +cov_thresh = []; + +alpha = []; + +line_type = [ "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2", + "SL1L2", "SL1L2", "SL1L2", "SL1L2" + ]; + +column = [ "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE", + "RMSE", "RMSE", "RMSE", "RMSE" + ]; + +weight = [ 4.0, 3.0, 2.0, 1.0, + 4.0, 3.0, 2.0, 1.0, + 4.0, 3.0, 2.0, 1.0, + 8.0, 6.0, 4.0, 2.0, + 8.0, 6.0, 4.0, 2.0, + 8.0, 6.0, 4.0, 2.0, + 8.0, 6.0, 4.0, 2.0, + 8.0, 6.0, 4.0, 2.0, + 4.0, 3.0, 2.0, 1.0, + 8.0, 6.0, 4.0, 2.0, + 4.0, 3.0, 2.0, 1.0, + 8.0, 6.0, 4.0, 2.0 + ]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +out_alpha = 0.05; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +hss_ec_value = NA; +rank_corr_flag = FALSE; +vif_flag = FALSE; +tmp_dir = "/tmp"; +version = "V10.1.0"; + +//////////////////////////////////////////////////////////////////////////////// From fde1bb8c3e4888254ae8d4f2d101b1ea65024a54 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 16 Jul 2021 17:02:08 -0600 Subject: [PATCH 03/60] Per #1788, add hooks for computing -job cbs_score. --- met/src/libcode/vx_analysis_util/stat_job.cc | 4 +- met/src/libcode/vx_analysis_util/stat_job.h | 16 ++-- .../tools/core/stat_analysis/stat_analysis.cc | 81 ++++++++-------- .../tools/core/stat_analysis/stat_analysis.h | 5 +- .../core/stat_analysis/stat_analysis_job.cc | 95 ++++++------------- .../core/stat_analysis/stat_analysis_job.h | 3 - 6 files changed, 85 insertions(+), 119 deletions(-) diff --git a/met/src/libcode/vx_analysis_util/stat_job.cc b/met/src/libcode/vx_analysis_util/stat_job.cc index dcd03688dc..397bd275e9 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.cc +++ b/met/src/libcode/vx_analysis_util/stat_job.cc @@ -2893,8 +2893,10 @@ STATJobType string_to_statjobtype(const char *str) { else if(strcasecmp(str, statjobtype_str[4]) == 0) t = stat_job_go_index; else if(strcasecmp(str, statjobtype_str[5]) == 0) - t = stat_job_ss_index; + t = stat_job_cbs_score; else if(strcasecmp(str, statjobtype_str[6]) == 0) + t = stat_job_ss_index; + else if(strcasecmp(str, statjobtype_str[7]) == 0) t = stat_job_ramp; else t = no_stat_job_type; diff --git a/met/src/libcode/vx_analysis_util/stat_job.h b/met/src/libcode/vx_analysis_util/stat_job.h index 72254bef5c..c804d13b11 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.h +++ b/met/src/libcode/vx_analysis_util/stat_job.h @@ -6,8 +6,6 @@ // ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA // *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* - - //////////////////////////////////////////////////////////////////////// #ifndef __STAT_JOB_H__ @@ -76,19 +74,21 @@ enum STATJobType { stat_job_go_index = 4, // Compute the GO Index. - stat_job_ss_index = 5, // Compute the Skill Score Index. + stat_job_cbs_score = 5, // Compute the CBS Score. + + stat_job_ss_index = 6, // Compute the Skill Score Index. - stat_job_ramp = 6, // Time-series ramp evaluation. + stat_job_ramp = 7, // Time-series ramp evaluation. - no_stat_job_type = 7 // Default value + no_stat_job_type = 8 // Default value }; -static const int n_statjobtypes = 8; +static const int n_statjobtypes = 9; static const char * const statjobtype_str[n_statjobtypes] = { "filter", "summary", "aggregate", - "aggregate_stat", "go_index", "ss_index", - "ramp", "NA" + "aggregate_stat", "go_index", "cbs_score", + "ss_index", "ramp", "NA" }; extern const char *statjobtype_to_string(const STATJobType); diff --git a/met/src/tools/core/stat_analysis/stat_analysis.cc b/met/src/tools/core/stat_analysis/stat_analysis.cc index 8c1df6d2d6..8ee42dac56 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis.cc @@ -41,6 +41,7 @@ // percentile thresholds. // 012 04/25/21 Halley Gotway Replace pickle files for temporary // ascii. +// 013 07/16/21 Halley Gotway MET #1788 Add CBS Score. // //////////////////////////////////////////////////////////////////////// @@ -433,8 +434,8 @@ void set_out_file(const char *path) { void process_search_dirs() { int n, i, j, max_len, n_read, n_keep; - MetConfig go_conf; - STATAnalysisJob go_job; + MetConfig ss_index_conf; + STATAnalysisJob ss_index_job; // // Initialize @@ -456,40 +457,41 @@ void process_search_dirs() { } // - // Apply the GO Index filtering criteria for a command line job. + // Apply the GO Index or CBS Score filtering criteria. // - if(default_job.job_type == stat_job_go_index) { + if(default_job.job_type == stat_job_go_index || + default_job.job_type == stat_job_cbs_score) { - mlog << Debug(1) << "GO Index Config File: " - << replace_path(go_index_config_file) << "\n"; + ConcatString config_file = + (default_job.job_type == stat_job_go_index ? + replace_path(go_index_config_file) : + replace_path(cbs_score_config_file)); + + mlog << Debug(1) << "Skill Score Index Config File: " + << config_file << "\n"; // - // Read config file constants followed by the config file which - // defines the GO Index. + // Read the config files for the constants and the skill score index. // - go_conf.read(replace_path(config_const_filename).c_str()); - go_conf.read(replace_path(go_index_config_file).c_str()); + ss_index_conf.read(replace_path(config_const_filename).c_str()); + ss_index_conf.read(config_file.c_str()); // - // Parse the contents of the GO Index config file into the - // search job. + // Parse the Skill Score Index config file into the search job. // - set_job_from_config(go_conf, go_job); + set_job_from_config(ss_index_conf, ss_index_job); // - // Amend the default job with GO Index filtering criteria. + // Amend the default job with Skill Score Index filtering criteria. // - default_job.parse_job_command(go_job.get_jobstring().c_str()); - - } // end if go_index + default_job.parse_job_command(ss_index_job.get_jobstring().c_str()); + } // // Open up the temp file for storing the intermediate STAT line data // - open_temps(); - // // Go through each input file // @@ -633,9 +635,9 @@ return; //////////////////////////////////////////////////////////////////////// void process_job(const char * jobstring, int n_job) { - STATAnalysisJob job, go_job; + STATAnalysisJob job, ss_index_job; ConcatString full_jobstring; - MetConfig go_conf; + MetConfig ss_index_conf; mlog << Debug(4) << "process_job(jobstring): " << jobstring << "\n"; @@ -659,35 +661,38 @@ void process_job(const char * jobstring, int n_job) { } // - // Special processing for the GO Index job. + // Special processing for the GO Index and CBS Score jobs. // - if(job.job_type == stat_job_go_index) { + if(default_job.job_type == stat_job_go_index || + default_job.job_type == stat_job_cbs_score) { + + ConcatString config_file = + (default_job.job_type == stat_job_go_index ? + replace_path(go_index_config_file) : + replace_path(cbs_score_config_file)); - mlog << Debug(1) << "GO Index Config File: " - << replace_path(go_index_config_file) << "\n"; + mlog << Debug(1) << "Skill Score Index Config File: " + << config_file << "\n"; // - // Read config file constants followed by the config file which - // defines the GO Index. + // Read the config files for the constants and the skill score index. // - go_conf.read(replace_path(config_const_filename).c_str()); - go_conf.read(replace_path(go_index_config_file).c_str()); + ss_index_conf.read(replace_path(config_const_filename).c_str()); + ss_index_conf.read(config_file.c_str()); // - // Parse the contents of the GO Index config file into the - // search job. + // Parse the Skill Score Index config file into the search job. // - set_job_from_config(go_conf, go_job); + set_job_from_config(ss_index_conf, ss_index_job); // - // Amend the current job with GO Index filtering criteria + // Amend the current job with Skill Score Index filtering criteria. // mlog << Debug(4) - << "\nAmending Job " << n_job << " with GO Index configuration file: " - << replace_path(go_index_config_file) << "\n"; - job.parse_job_command(go_job.get_jobstring().c_str()); - - } // end if go_index + << "\nAmending Job " << n_job << " with Skill Score Index configuration file: " + << config_file << "\n"; + job.parse_job_command(ss_index_job.get_jobstring().c_str()); + } // // Amend the current job using any command line options diff --git a/met/src/tools/core/stat_analysis/stat_analysis.h b/met/src/tools/core/stat_analysis/stat_analysis.h index c07abfa476..ee8250dbcd 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis.h +++ b/met/src/tools/core/stat_analysis/stat_analysis.h @@ -55,7 +55,10 @@ static const char * default_config_filename = static const char * go_index_config_file = "MET_BASE/config/STATAnalysisConfig_GO_Index"; - + +static const char * cbs_score_config_file = + "MET_BASE/config/STATAnalysisConfig_CBS_Score"; + //////////////////////////////////////////////////////////////////////// // // Variables for Command Line Arguments diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index 451b6d785f..b53a958f8e 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -226,9 +226,7 @@ void do_job(const ConcatString &jobstring, STATAnalysisJob &job, break; case(stat_job_go_index): - do_job_go_index(jobstring, f, job, n_in, n_out, sa_out); - break; - + case(stat_job_cbs_score): case(stat_job_ss_index): do_job_ss_index(jobstring, f, job, n_in, n_out, sa_out); break; @@ -3934,10 +3932,21 @@ void write_job_ramp_cols(const STATAnalysisJob &job, AsciiTable &at, //////////////////////////////////////////////////////////////////////// // -// The do_job_go_index() routine is used to compute the GO Index. -// The GO Index is a special case of the Skill Score Index consisting -// of a predefined set of variables, levels, lead times, statistics, -// and weights. +// The do_job_ss_index() routine is used to compute the GO Index, +// CBS Score, or generalized Skill Score Index. This job can be +// configured to compute a weighted average of skill scores derived +// from a configurable set of variables, levels, lead times, and +// statistics. The skill score index is computed using two models, +// a forecast model and a reference model. For each statistic in +// the index, a skill score is computed as: +// SS = 1 - (S[model]*S[model])/(S[reference]*S[reference]) +// Where S is the statistic. +// Next, a weighted average is computed over all the skill scores. +// Lastly, an index value is computed as: +// Index = sqrt(1/(1-SS[avg])) +// Where SS[avg] is the weighted average of skill scores. +// +// GO Index definition: // For lead times of 12, 24, 36, and 48 hours, it contains RMSE for: // - Wind Speed at the surface(b), 850(a), 400(a), 250(a) mb // - Dewpoint Temperature at the surface(b), 850(b), 700(b), 400(b) mB @@ -3950,72 +3959,19 @@ void write_job_ramp_cols(const STATAnalysisJob &job, AsciiTable &at, // //////////////////////////////////////////////////////////////////////// -void do_job_go_index(const ConcatString &jobstring, LineDataFile &f, +void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, STATAnalysisJob &job, int &n_in, int &n_out, ofstream *sa_out) { - double go_index; + double ss_index; + ConcatString cs, ss_index_cs; AsciiTable out_at; // - // Compute the GO Index as a special case of the Skill Score Index - // - go_index = compute_ss_index(jobstring, f, job, n_in, n_out); - - // - // Check for no matching STAT lines - // - if(n_out == 0) { - mlog << Warning << "\ndo_job_go_index() -> " - << "no matching STAT lines found for job: " << jobstring - << "\n\n"; - return; - } - - // - // Get the column names - // - out_at.set_size(2, 2); - setup_table(out_at, 1, job.get_precision()); - out_at.set_entry(0, 0, "COL_NAME:"); - write_header_row(job_go_columns, n_job_go_columns, 0, out_at, 0, 1); - - // - // Write the data row + // Determine the job type // - out_at.set_entry(1, 0, "GO_INDEX:"); - out_at.set_entry(1, 1, go_index); - - // - // Write the Ascii Table and the job command line - // - write_jobstring(jobstring, sa_out); - write_table(out_at, sa_out); - - return; -} - -//////////////////////////////////////////////////////////////////////// -// -// The do_job_ss_index() routine is used to compute the generalized -// Skill Score Index. This job can be configured to compute a weighted -// average of skill scores derived from a configurable set of variables, -// levels, lead times, and statistics. The skill score index is -// computed using two models, a forecast model and a reference model. -// For each statistic in the index, a skill score is computed as: -// SS = 1 - (S[model]*S[model])/(S[reference]*S[reference]) -// Where S is the statistic. -// Next, a weighted average is computed over all the skill scores. -// Lastly, an index value is computed as: -// Index = sqrt(1/(1-SS[avg])) -// Where SS[avg] is the weighted average of skill scores. -// -//////////////////////////////////////////////////////////////////////// - -void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, - STATAnalysisJob &job, int &n_in, int &n_out, - ofstream *sa_out) { - double ss_index; - AsciiTable out_at; + if(job.job_type == stat_job_go_index) ss_index_cs = "GO_INDEX"; + else if(job.job_type == stat_job_cbs_score) ss_index_cs = "CBS_SCORE"; + else ss_index_cs = "SS_INDEX"; // // Compute the Skill Score Index @@ -4040,10 +3996,13 @@ void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, out_at.set_entry(0, 0, "COL_NAME:"); write_header_row(job_ss_columns, n_job_ss_columns, 0, out_at, 0, 1); +// JHG, write stat output here! + // // Write the data row // - out_at.set_entry(1, 0, "SS_INDEX:"); + cs << cs_erase << ss_index_cs << ":"; + out_at.set_entry(1, 0, cs); out_at.set_entry(1, 1, ss_index); // diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.h b/met/src/tools/core/stat_analysis/stat_analysis_job.h index e03c74330d..dbda9011ba 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.h +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.h @@ -68,9 +68,6 @@ extern void do_job_aggr_stat(const ConcatString &, LineDataFile &, STATAnalysisJob &, int &, int &, ofstream *, const ConcatString &, gsl_rng *); -extern void do_job_go_index(const ConcatString &, LineDataFile &, - STATAnalysisJob &, int &, int &, ofstream *); - extern void do_job_ss_index(const ConcatString &, LineDataFile &, STATAnalysisJob &, int &, int &, ofstream *); From b2b3d0871ab9f70db501cfd0325e14e1000681d3 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 19 Jul 2021 16:30:40 -0600 Subject: [PATCH 04/60] Per #1788, since we're adding a new SSIDX line type, the columns for that line type replace the job_ss_columns and job_go_columns arrays. --- met/src/basic/vx_util/stat_column_defs.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/met/src/basic/vx_util/stat_column_defs.h b/met/src/basic/vx_util/stat_column_defs.h index d34fa22fdd..1eb2778e96 100644 --- a/met/src/basic/vx_util/stat_column_defs.h +++ b/met/src/basic/vx_util/stat_column_defs.h @@ -330,14 +330,6 @@ static const char * job_summary_columns [] = { "WMO_TYPE", "WMO_MEAN", "WMO_WEIGHTED_MEAN" }; -static const char * job_go_columns [] = { - "GO_INDEX" -}; - -static const char * job_ss_columns [] = { - "SS_INDEX" -}; - static const char * job_wdir_columns [] = { "TOTAL", "FBAR", "OBAR", "ME", "MAE" @@ -397,8 +389,6 @@ static const int n_dmap_columns = sizeof(dmap_columns)/sizeof(*dmap_colu static const int n_isc_columns = sizeof(isc_columns)/sizeof(*isc_columns); static const int n_job_summary_columns = sizeof(job_summary_columns)/sizeof(*job_summary_columns); -static const int n_job_go_columns = sizeof(job_go_columns)/sizeof(*job_go_columns); -static const int n_job_ss_columns = sizeof(job_ss_columns)/sizeof(*job_ss_columns); static const int n_job_wdir_columns = sizeof(job_wdir_columns)/sizeof(*job_wdir_columns); static const int n_job_ramp_columns = sizeof(job_ramp_columns)/sizeof(*job_ramp_columns); static const int n_job_ramp_mpr_columns = sizeof(job_ramp_mpr_columns)/sizeof(*job_ramp_mpr_columns); From 128c41ef827c30faa59b2cbe188b7850c2e7e14e Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 22 Jul 2021 10:36:19 -0600 Subject: [PATCH 05/60] Per #1788, rename cbs_score to cbs_index per UK Met Office instruction on 7/22/21. This convention is also consistent with the existing go_index and ss_index jobs. --- met/src/libcode/vx_analysis_util/stat_job.cc | 2 +- met/src/libcode/vx_analysis_util/stat_job.h | 4 ++-- met/src/tools/core/stat_analysis/stat_analysis.cc | 14 +++++++------- met/src/tools/core/stat_analysis/stat_analysis.h | 4 ++-- .../tools/core/stat_analysis/stat_analysis_job.cc | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/met/src/libcode/vx_analysis_util/stat_job.cc b/met/src/libcode/vx_analysis_util/stat_job.cc index 397bd275e9..c7146bae62 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.cc +++ b/met/src/libcode/vx_analysis_util/stat_job.cc @@ -2893,7 +2893,7 @@ STATJobType string_to_statjobtype(const char *str) { else if(strcasecmp(str, statjobtype_str[4]) == 0) t = stat_job_go_index; else if(strcasecmp(str, statjobtype_str[5]) == 0) - t = stat_job_cbs_score; + t = stat_job_cbs_index; else if(strcasecmp(str, statjobtype_str[6]) == 0) t = stat_job_ss_index; else if(strcasecmp(str, statjobtype_str[7]) == 0) diff --git a/met/src/libcode/vx_analysis_util/stat_job.h b/met/src/libcode/vx_analysis_util/stat_job.h index c804d13b11..b075ffc9ed 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.h +++ b/met/src/libcode/vx_analysis_util/stat_job.h @@ -74,7 +74,7 @@ enum STATJobType { stat_job_go_index = 4, // Compute the GO Index. - stat_job_cbs_score = 5, // Compute the CBS Score. + stat_job_cbs_index = 5, // Compute the CBS Index. stat_job_ss_index = 6, // Compute the Skill Score Index. @@ -87,7 +87,7 @@ static const int n_statjobtypes = 9; static const char * const statjobtype_str[n_statjobtypes] = { "filter", "summary", "aggregate", - "aggregate_stat", "go_index", "cbs_score", + "aggregate_stat", "go_index", "cbs_index", "ss_index", "ramp", "NA" }; diff --git a/met/src/tools/core/stat_analysis/stat_analysis.cc b/met/src/tools/core/stat_analysis/stat_analysis.cc index 8ee42dac56..04914f9ec9 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis.cc @@ -41,7 +41,7 @@ // percentile thresholds. // 012 04/25/21 Halley Gotway Replace pickle files for temporary // ascii. -// 013 07/16/21 Halley Gotway MET #1788 Add CBS Score. +// 013 07/16/21 Halley Gotway MET #1788 Add CBS Index. // //////////////////////////////////////////////////////////////////////// @@ -457,15 +457,15 @@ void process_search_dirs() { } // - // Apply the GO Index or CBS Score filtering criteria. + // Apply the GO Index or CBS Index filtering criteria. // if(default_job.job_type == stat_job_go_index || - default_job.job_type == stat_job_cbs_score) { + default_job.job_type == stat_job_cbs_index) { ConcatString config_file = (default_job.job_type == stat_job_go_index ? replace_path(go_index_config_file) : - replace_path(cbs_score_config_file)); + replace_path(cbs_index_config_file)); mlog << Debug(1) << "Skill Score Index Config File: " << config_file << "\n"; @@ -661,15 +661,15 @@ void process_job(const char * jobstring, int n_job) { } // - // Special processing for the GO Index and CBS Score jobs. + // Special processing for the GO Index and CBS Index jobs. // if(default_job.job_type == stat_job_go_index || - default_job.job_type == stat_job_cbs_score) { + default_job.job_type == stat_job_cbs_index) { ConcatString config_file = (default_job.job_type == stat_job_go_index ? replace_path(go_index_config_file) : - replace_path(cbs_score_config_file)); + replace_path(cbs_index_config_file)); mlog << Debug(1) << "Skill Score Index Config File: " << config_file << "\n"; diff --git a/met/src/tools/core/stat_analysis/stat_analysis.h b/met/src/tools/core/stat_analysis/stat_analysis.h index ee8250dbcd..bd70bb19b9 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis.h +++ b/met/src/tools/core/stat_analysis/stat_analysis.h @@ -56,8 +56,8 @@ static const char * default_config_filename = static const char * go_index_config_file = "MET_BASE/config/STATAnalysisConfig_GO_Index"; -static const char * cbs_score_config_file = - "MET_BASE/config/STATAnalysisConfig_CBS_Score"; +static const char * cbs_index_config_file = + "MET_BASE/config/STATAnalysisConfig_CBS_Index"; //////////////////////////////////////////////////////////////////////// // diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index b53a958f8e..06fff774df 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -226,7 +226,7 @@ void do_job(const ConcatString &jobstring, STATAnalysisJob &job, break; case(stat_job_go_index): - case(stat_job_cbs_score): + case(stat_job_cbs_index): case(stat_job_ss_index): do_job_ss_index(jobstring, f, job, n_in, n_out, sa_out); break; @@ -3933,7 +3933,7 @@ void write_job_ramp_cols(const STATAnalysisJob &job, AsciiTable &at, //////////////////////////////////////////////////////////////////////// // // The do_job_ss_index() routine is used to compute the GO Index, -// CBS Score, or generalized Skill Score Index. This job can be +// CBS Index, or generalized Skill Score Index. This job can be // configured to compute a weighted average of skill scores derived // from a configurable set of variables, levels, lead times, and // statistics. The skill score index is computed using two models, @@ -3970,7 +3970,7 @@ void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, // Determine the job type // if(job.job_type == stat_job_go_index) ss_index_cs = "GO_INDEX"; - else if(job.job_type == stat_job_cbs_score) ss_index_cs = "CBS_SCORE"; + else if(job.job_type == stat_job_cbs_index) ss_index_cs = "CBS_INDEX"; else ss_index_cs = "SS_INDEX"; // From 3084507a6d3946218485ecad0ac073d332e8dc4a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 23 Jul 2021 15:32:59 -0600 Subject: [PATCH 06/60] Per #1788 rename STATAnalysisConfig_CBS_Score to STATAnalysisConfig_CBS_Index --- ...TATAnalysisConfig_CBS_Score => STATAnalysisConfig_CBS_Index} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename met/data/config/{STATAnalysisConfig_CBS_Score => STATAnalysisConfig_CBS_Index} (98%) diff --git a/met/data/config/STATAnalysisConfig_CBS_Score b/met/data/config/STATAnalysisConfig_CBS_Index similarity index 98% rename from met/data/config/STATAnalysisConfig_CBS_Score rename to met/data/config/STATAnalysisConfig_CBS_Index index 8775d5ae37..62e2957674 100644 --- a/met/data/config/STATAnalysisConfig_CBS_Score +++ b/met/data/config/STATAnalysisConfig_CBS_Index @@ -1,6 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // -// STAT-Analysis configuration file used to compute the CBS Score. +// STAT-Analysis configuration file used to compute the CBS Index. // // WARNING: This file should not be modified. // From df77bae1be3defc2de977701a5b925388c8f29ee Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 23 Jul 2021 16:24:19 -0600 Subject: [PATCH 07/60] Per #1788, update Makefile to handle name change. --- met/data/config/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/data/config/Makefile.am b/met/data/config/Makefile.am index 0ab2bd0927..c73bb4cadc 100644 --- a/met/data/config/Makefile.am +++ b/met/data/config/Makefile.am @@ -35,7 +35,7 @@ config_DATA = \ SeriesAnalysisConfig_default \ STATAnalysisConfig_default \ STATAnalysisConfig_GO_Index \ - STATAnalysisConfig_CBS_Score \ + STATAnalysisConfig_CBS_Index \ TCPairsConfig_default \ TCRMWConfig_default \ RMWAnalysisConfig_default \ From 2fc2880ab4ef76c7cba7e4e35572370cdaecd410 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 2 Aug 2021 16:51:55 -0600 Subject: [PATCH 08/60] Per #1788, add support for SSIDX stat line type. --- met/src/basic/vx_config/config_util.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/met/src/basic/vx_config/config_util.cc b/met/src/basic/vx_config/config_util.cc index c55fd819ae..ffa3f28a34 100644 --- a/met/src/basic/vx_config/config_util.cc +++ b/met/src/basic/vx_config/config_util.cc @@ -2327,10 +2327,11 @@ const char * statlinetype_to_string(const STATLineType t) { case(stat_relp): s = stat_relp_str; break; case(stat_genmpr): s = stat_genmpr_str; break; + case(stat_ssidx): s = stat_ssidx_str; break; case(stat_header): s = stat_header_str; break; - case(no_stat_line_type): s = stat_na_str; break; - default: s = (const char *) 0; break; + case(no_stat_line_type): + default: s = stat_na_str; break; } return(s); @@ -2388,6 +2389,7 @@ STATLineType string_to_statlinetype(const char *s) { else if(strcasecmp(s, stat_relp_str) == 0) t = stat_relp; else if(strcasecmp(s, stat_genmpr_str) == 0) t = stat_genmpr; + else if(strcasecmp(s, stat_ssidx_str) == 0) t = stat_ssidx; else if(strcasecmp(s, stat_header_str) == 0) t = stat_header; else t = no_stat_line_type; From 9e59ed532d2423fd8bbfab0f9f6896849d9eb84c Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 2 Aug 2021 16:52:27 -0600 Subject: [PATCH 09/60] Per #1788, define 4 output SSIDX columns. --- met/src/basic/vx_util/stat_column_defs.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/met/src/basic/vx_util/stat_column_defs.h b/met/src/basic/vx_util/stat_column_defs.h index 88f918ea20..f9b1b31bfa 100644 --- a/met/src/basic/vx_util/stat_column_defs.h +++ b/met/src/basic/vx_util/stat_column_defs.h @@ -1,4 +1,4 @@ -// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +/// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* // ** Copyright UCAR (c) 1992 - 2021 // ** University Corporation for Atmospheric Research (UCAR) // ** National Center for Atmospheric Research (NCAR) @@ -308,6 +308,14 @@ static const char * ssvar_columns [] = { "RMSE" }; +static const char * relp_columns [] = { + "TOTAL", "N_ENS", "RELP_" +}; + +static const char * ssidx_columns [] = { + "NAME", "N_TERM", "N_VLD", "SSINDEX" +}; + static const char * genmpr_columns [] = { "TOTAL", "INDEX", "STORM_ID", "AGEN_INIT", "AGEN_FHR", @@ -317,10 +325,6 @@ static const char * genmpr_columns [] = { "DEV_CAT", "OPS_CAT" }; -static const char * relp_columns [] = { - "TOTAL", "N_ENS", "RELP_" -}; - static const char * job_summary_columns [] = { "TOTAL", "MEAN", "MEAN_NCL", "MEAN_NCU", "MEAN_BCL", "MEAN_BCU", @@ -403,6 +407,8 @@ static const int n_orank_columns = sizeof(orank_columns)/sizeof(*orank_co static const int n_ssvar_columns = sizeof(ssvar_columns)/sizeof(*ssvar_columns); static const int n_relp_columns = sizeof(relp_columns)/sizeof(*relp_columns); +static const int n_ssidx_columns = sizeof(ssidx_columns)/sizeof(*ssidx_columns); + static const int n_genmpr_columns = sizeof(genmpr_columns)/sizeof(*genmpr_columns); //////////////////////////////////////////////////////////////////////// From 2b50bd54f01dec1fd27dd029643950d0526f4ea0 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 2 Aug 2021 16:53:21 -0600 Subject: [PATCH 10/60] Per #1788, define an SSIndexInfo struct for stashing/writing SS Index values. --- met/src/libcode/vx_statistics/met_stats.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/met/src/libcode/vx_statistics/met_stats.h b/met/src/libcode/vx_statistics/met_stats.h index 9f29299333..58bf2044d3 100644 --- a/met/src/libcode/vx_statistics/met_stats.h +++ b/met/src/libcode/vx_statistics/met_stats.h @@ -651,6 +651,18 @@ class DMAPInfo { void reset_options(); }; +//////////////////////////////////////////////////////////////////////// +// +// Structure to store the Skill Score Index +// +//////////////////////////////////////////////////////////////////////// + +struct SSIndexInfo { + ConcatString name; + int n_term, n_vld; + double ss_index; +}; + //////////////////////////////////////////////////////////////////////// // // Utility functions for parsing data from configuration files From 3c9b2e41da9a001f864efdcce5ad2a5454f4d743 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 2 Aug 2021 16:54:08 -0600 Subject: [PATCH 11/60] Per #1788, write the SSIDX columns. However we don't need to function to write the whole row since it's only written by Stat-Analysis. --- met/src/libcode/vx_stat_out/stat_columns.cc | 57 +++++---------------- met/src/libcode/vx_stat_out/stat_columns.h | 4 +- 2 files changed, 14 insertions(+), 47 deletions(-) diff --git a/met/src/libcode/vx_stat_out/stat_columns.cc b/met/src/libcode/vx_stat_out/stat_columns.cc index 00d2d195e2..1600dcb98e 100644 --- a/met/src/libcode/vx_stat_out/stat_columns.cc +++ b/met/src/libcode/vx_stat_out/stat_columns.cc @@ -1892,47 +1892,6 @@ void write_relp_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr, //////////////////////////////////////////////////////////////////////// -void write_ssidx_row(StatHdrColumns &shc, double ss_index, - STATOutputType out_type, - AsciiTable &stat_at, int &stat_row, - AsciiTable &txt_at, int &txt_row) { - int i; - - // SSIDX line type - shc.set_line_type(stat_ssidx_str); - - // Not Applicable - shc.set_interp_mthd(InterpMthd_None, - GridTemplateFactory::GridTemplate_None); - shc.set_interp_wdth(bad_data_int); - shc.set_fcst_thresh(na_str); - shc.set_obs_thresh(na_str); - shc.set_thresh_logic(SetLogic_None); - shc.set_cov_thresh(na_str); - shc.set_alpha(bad_data_double); - - // Write the header columns - write_header_cols(shc, stat_at, stat_row); - - // Write the data columns - write_ssidx_cols(ss_index, stat_at, stat_row, n_header_columns); - - // If requested, copy row to the text file - if(out_type == STATOutputType_Both) { - copy_ascii_table_row(stat_at, stat_row, txt_at, txt_row); - - // Increment the text row counter - txt_row++; - } - - // Increment the STAT row counter - stat_row++; - - return; -} - -//////////////////////////////////////////////////////////////////////// - void write_header_cols(const StatHdrColumns &shc, AsciiTable &at, int r) { ConcatString cs; @@ -4157,16 +4116,26 @@ void write_relp_cols(const PairDataEnsemble *pd_ptr, //////////////////////////////////////////////////////////////////////// -void write_ssidx_cols(double ss_index, +void write_ssidx_cols(SSIndexInfo ssidx_info, AsciiTable &at, int r, int c) { // // Skill Score Index // Dump out the SSIDX line: + // NAME, N_TERM, N_VLD, // SS_INDEX // - at.set_entry(r, c+0, // Skill Score Index - ss_index); + at.set_entry(r, c+0, // Skill score index name + ssidx_info.name); + + at.set_entry(r, c+1, // Number of terms + ssidx_info.n_term); + + at.set_entry(r, c+2, // Number of valid terms + ssidx_info.n_vld); + + at.set_entry(r, c+3, // Skill score index value + ssidx_info.ss_index); return; } diff --git a/met/src/libcode/vx_stat_out/stat_columns.h b/met/src/libcode/vx_stat_out/stat_columns.h index 762c959464..bde05f3ffb 100644 --- a/met/src/libcode/vx_stat_out/stat_columns.h +++ b/met/src/libcode/vx_stat_out/stat_columns.h @@ -120,8 +120,6 @@ extern void write_ssvar_row (StatHdrColumns &, const PairDataEnsemble *, double, AsciiTable &, int &, AsciiTable &, int &); extern void write_relp_row (StatHdrColumns &, const PairDataEnsemble *, STATOutputType, AsciiTable &, int &, AsciiTable &, int &); -extern void write_ssidx_row (StatHdrColumns &, double, STATOutputType, - AsciiTable &, int &, AsciiTable &, int &); //////////////////////////////////////////////////////////////////////// @@ -191,7 +189,7 @@ extern void write_ssvar_cols (const PairDataEnsemble *, int, double, AsciiTable &, int, int); extern void write_relp_cols (const PairDataEnsemble *, AsciiTable &, int, int); -extern void write_ssidx_cols (double, +extern void write_ssidx_cols (const SSIndexInfo, AsciiTable &, int, int); // Setup column justification for STAT AsciiTable objects From 7f2f7294ad4262494fc035aec3bc7a9a3763af03 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 2 Aug 2021 16:55:35 -0600 Subject: [PATCH 12/60] Per #1788, create an AggrSSIndexInfo struct to keep track of unique stat header column input values. --- met/src/tools/core/stat_analysis/aggr_stat_line.cc | 4 ++-- met/src/tools/core/stat_analysis/aggr_stat_line.h | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.cc b/met/src/tools/core/stat_analysis/aggr_stat_line.cc index e6f621feb0..593f93b45c 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -466,7 +466,7 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, // INTERP_PNTS css = write_css(interp_pnts); - if(interp_pnts.n() > 1) { + if(interp_pnts.n() == 0 || interp_pnts.n() > 1) { mlog << Warning << "For case \"" << cur_case << "\", found " << interp_pnts.n() @@ -502,7 +502,7 @@ StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, // ALPHA css = write_css(alpha); - if(alpha.n() > 1) { + if(alpha.n() == 0 || alpha.n() > 1) { mlog << Warning << "For case \"" << cur_case << "\", found " << alpha.n() diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.h b/met/src/tools/core/stat_analysis/aggr_stat_line.h index 166dde307e..4e8d5e0461 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.h +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.h @@ -160,6 +160,11 @@ struct AggrRPSInfo { RPSInfo rps_info; }; +struct AggrSSIndexInfo { + StatHdrInfo hdr; + SSIndexInfo info; +}; + // Define struct used to perform comparisons for SSVAR bins struct ssvar_bin_cmp { bool operator()(const ConcatString & cs1, const ConcatString & cs2) const { From f60a2bf8fdeba24a5e3fde700384e09514563038 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 2 Aug 2021 16:56:31 -0600 Subject: [PATCH 13/60] Per #1788, update the stat-analysis job code to make use of the new structs. --- met/src/libcode/vx_analysis_util/stat_job.cc | 2 + .../core/stat_analysis/stat_analysis_job.cc | 197 +++++++++++------- .../core/stat_analysis/stat_analysis_job.h | 9 +- 3 files changed, 133 insertions(+), 75 deletions(-) diff --git a/met/src/libcode/vx_analysis_util/stat_job.cc b/met/src/libcode/vx_analysis_util/stat_job.cc index c7146bae62..9b0169e876 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.cc +++ b/met/src/libcode/vx_analysis_util/stat_job.cc @@ -1969,6 +1969,7 @@ void STATAnalysisJob::setup_stat_file(int n_row, int n) { case stat_orank: c = n_orank_columns; break; case stat_ssvar: c = n_ssvar_columns; break; case stat_genmpr: c = n_genmpr_columns; break; + case stat_ssidx: c = n_ssidx_columns; break; default: mlog << Error << "\nSTATAnalysisJob::setup_stat_file() -> " << "unexpected stat line type \"" << statlinetype_to_string(cur_lt) @@ -2040,6 +2041,7 @@ void STATAnalysisJob::setup_stat_file(int n_row, int n) { case stat_orank: write_header_row (orank_columns, n_orank_columns, 1, stat_at, 0, 0); break; case stat_ssvar: write_header_row (ssvar_columns, n_ssvar_columns, 1, stat_at, 0, 0); break; case stat_genmpr: write_header_row (genmpr_columns, n_genmpr_columns, 1, stat_at, 0, 0); break; + case stat_ssidx: write_header_row (ssidx_columns, n_ssidx_columns, 1, stat_at, 0, 0); break; // // Write only header columns for unspecified line type diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index 06fff774df..a094ff5554 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -3930,6 +3930,63 @@ void write_job_ramp_cols(const STATAnalysisJob &job, AsciiTable &at, return; } +//////////////////////////////////////////////////////////////////////// + +void write_job_ss_index(STATAnalysisJob &job, + AggrSSIndexInfo &ssidx_info, + AsciiTable &at) { + int n_row, n_col, r, c; + StatHdrColumns shc; + + // + // Setup the output table + // + n_row = 2; + n_col = 1 + n_ssidx_columns; + + // + // Prepare the output + // + if(job.stat_out) { + job.setup_stat_file(n_row, 0); + } + else { + write_job_aggr_hdr(job, n_row, n_col, at); + write_header_row(ssidx_columns, n_ssidx_columns, 0, at, 0, 1); + } + + mlog << Debug(2) << "Computing output for 1 case(s).\n"; + + // + // Write the output STAT header columns + // + ConcatString cur_case(""); + shc = ssidx_info.hdr.get_shc(cur_case, job.by_column, + job.hdr_name, job.hdr_value, stat_ssidx); + + // + // Initialize + // + r = 1; + c = 0; + + // + // SSIDX output line + // + if(job.stat_out) { + write_header_cols(shc, job.stat_at, job.stat_row); + write_ssidx_cols(ssidx_info.info, job.stat_at, + job.stat_row++, n_header_columns); + } + else { + at.set_entry(r, c++, "SSIDX:"); + write_case_cols(cur_case, at, r, c); + write_ssidx_cols(ssidx_info.info, at, r++, c); + } + + return; +} + //////////////////////////////////////////////////////////////////////// // // The do_job_ss_index() routine is used to compute the GO Index, @@ -3962,21 +4019,25 @@ void write_job_ramp_cols(const STATAnalysisJob &job, AsciiTable &at, void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, STATAnalysisJob &job, int &n_in, int &n_out, ofstream *sa_out) { - double ss_index; - ConcatString cs, ss_index_cs; + AggrSSIndexInfo ssidx_info; AsciiTable out_at; + // + // Store the output line type + // + job.out_line_type.add(stat_ssidx_str); + // // Determine the job type // - if(job.job_type == stat_job_go_index) ss_index_cs = "GO_INDEX"; - else if(job.job_type == stat_job_cbs_index) ss_index_cs = "CBS_INDEX"; - else ss_index_cs = "SS_INDEX"; + if(job.job_type == stat_job_go_index) ssidx_info.info.name = "GO_INDEX"; + else if(job.job_type == stat_job_cbs_index) ssidx_info.info.name = "CBS_INDEX"; + else ssidx_info.info.name = "SS_INDEX"; // // Compute the Skill Score Index // - ss_index = compute_ss_index(jobstring, f, job, n_in, n_out); + compute_ss_index(f, job, ssidx_info, n_in, n_out); // // Check for no matching STAT lines @@ -3989,21 +4050,9 @@ void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, } // - // Get the column names + // Write the output // - out_at.set_size(2, 2); - setup_table(out_at, 1, job.get_precision()); - out_at.set_entry(0, 0, "COL_NAME:"); - write_header_row(job_ss_columns, n_job_ss_columns, 0, out_at, 0, 1); - -// JHG, write stat output here! - - // - // Write the data row - // - cs << cs_erase << ss_index_cs << ":"; - out_at.set_entry(1, 0, cs); - out_at.set_entry(1, 1, ss_index); + write_job_ss_index(job, ssidx_info, out_at); // // Write the Ascii Table and the job command line @@ -4202,16 +4251,17 @@ void write_line(const ConcatString &str, ofstream *sa_out) { //////////////////////////////////////////////////////////////////////// -double compute_ss_index(const ConcatString &jobstring, LineDataFile &f, - STATAnalysisJob &job, int &n_in, int &n_out) { +void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, + AggrSSIndexInfo &ssidx_info, + int &n_in, int &n_out) { STATLine line; SL1L2Info si; TTContingencyTable ct; CNTInfo fcst_cnt, ref_cnt; bool keep; - int i, n_terms; + int i, n_term, n_vld; double fcst_stat, ref_stat, ss, ss_sum, weight_sum; - double ss_avg, ss_index; + double ss_avg; // // Check that the -model option has been supplied exactly 2 times. @@ -4221,37 +4271,34 @@ double compute_ss_index(const ConcatString &jobstring, LineDataFile &f, mlog << Error << "\ncompute_ss_index() -> " << "this job may only be called when the \"-model\" option " << "has been used exactly twice to specify the forecast " - << "model followed by the reference model: " - << jobstring << "\n\n"; + << "model followed by the reference model.\n\n"; throw(1); } // // Use the length of the fcst_var array to infer the number of terms. // - if((n_terms = job.fcst_var.n()) < 1) { + if((n_term = job.fcst_var.n()) < 1) { mlog << Error << "\ncompute_ss_index() -> " << "you must define the Skill Score Index to be computed " << "using the \"-fcst_var\", \"-fcst_lev\", \"-fcst_lead\", " - << "\"-line_type\", \"-column\", and \"-weight\" options: " - << jobstring << "\n\n"; + << "\"-line_type\", \"-column\", and \"-weight\" options.\n\n"; throw(1); } // // Check that the required elements are of the same length. // - if(n_terms != job.fcst_lev.n() || - n_terms != job.fcst_lead.n() || - n_terms != job.line_type.n() || - n_terms != job.column.n() || - n_terms != job.weight.n()) { + if(n_term != job.fcst_lev.n() || + n_term != job.fcst_lead.n() || + n_term != job.line_type.n() || + n_term != job.column.n() || + n_term != job.weight.n()) { mlog << Error << "\ncompute_ss_index() -> " << "all filtering parameters for defining the Skill Score " << "Index must be of the same length. Check \"-fcst_var\", " << "\"-fcst_lev\", \"-fcst_lead\", \"-line_type\", " - << "\"-column\", and \"-weight\" options: " - << jobstring << "\n\n"; + << "\"-column\", and \"-weight\" options.\n\n"; throw(1); } @@ -4260,27 +4307,26 @@ double compute_ss_index(const ConcatString &jobstring, LineDataFile &f, // Separate arrays for the forecast and reference models. // STATAnalysisJob *fcst_job = (STATAnalysisJob *) 0, *ref_job = (STATAnalysisJob *) 0; - fcst_job = new STATAnalysisJob [n_terms]; - ref_job = new STATAnalysisJob [n_terms]; + fcst_job = new STATAnalysisJob [n_term]; + ref_job = new STATAnalysisJob [n_term]; // // Define arrays of objects to store the partial sums or contingency // table counts for each term in the Skill Score Index. // - SL1L2Info *fcst_si = (SL1L2Info *) 0, *ref_si = (SL1L2Info *) 0; - CTSInfo *fcst_cts = (CTSInfo *) 0, *ref_cts = (CTSInfo *) 0; - fcst_si = new SL1L2Info [n_terms]; - ref_si = new SL1L2Info [n_terms]; - fcst_cts = new CTSInfo [n_terms]; - ref_cts = new CTSInfo [n_terms]; - + SL1L2Info *fcst_si = (SL1L2Info *) 0, *ref_si = (SL1L2Info *) 0; + CTSInfo *fcst_cts = (CTSInfo *) 0, *ref_cts = (CTSInfo *) 0; + fcst_si = new SL1L2Info [n_term]; + ref_si = new SL1L2Info [n_term]; + fcst_cts = new CTSInfo [n_term]; + ref_cts = new CTSInfo [n_term]; // // Define array of line types to be aggregated for each term in the // Skill Score Index. // STATLineType *job_lt = (STATLineType *) 0; - job_lt = new STATLineType [n_terms]; + job_lt = new STATLineType [n_term]; // // Arrays to keep track of the number of stat lines per term @@ -4293,7 +4339,7 @@ double compute_ss_index(const ConcatString &jobstring, LineDataFile &f, // // Set up the job for each term in the index. // - for(i=0; i " << "can't compute skill score for term " << i+1 << ".\n\n"; } + else { + n_vld++; + } } // end for i @@ -4724,12 +4772,17 @@ double compute_ss_index(const ConcatString &jobstring, LineDataFile &f, // // Compute the Skill Score Index value. // - if(is_bad_data(ss_avg) || - is_eq(ss_avg, 1.0)) ss_index = bad_data_double; - else ss_index = sqrt(1.0/(1.0 - ss_avg)); + ssidx_info.info.n_term = n_term; + ssidx_info.info.n_vld = n_vld; + if(is_bad_data(ss_avg) || is_eq(ss_avg, 1.0)) { + ssidx_info.info.ss_index = bad_data_double; + } + else { + ssidx_info.info.ss_index = sqrt(1.0/(1.0 - ss_avg)); + } mlog << Debug(3) << "Skill Score Index Weighted Average = " << ss_avg << "\n" - << "Skill Score Index Value = " << ss_index << "\n"; + << "Skill Score Index Value = " << ssidx_info.info.ss_index << "\n"; // // Clean up allocated memory. @@ -4742,7 +4795,7 @@ double compute_ss_index(const ConcatString &jobstring, LineDataFile &f, if(ref_cts) { delete [] ref_cts; ref_cts = (CTSInfo *) 0; } if(job_lt) { delete [] job_lt; job_lt = (STATLineType *) 0; } - return(ss_index); + return; } //////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.h b/met/src/tools/core/stat_analysis/stat_analysis_job.h index dbda9011ba..5f67470758 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.h +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.h @@ -140,6 +140,9 @@ extern void write_job_ramp(STATAnalysisJob &, extern void write_job_ramp_cols(const STATAnalysisJob &, AsciiTable &, int &, int &); +extern void write_job_ss_index(STATAnalysisJob &, + AggrSSIndexInfo &, AsciiTable &); + //////////////////////////////////////////////////////////////////////// extern void setup_table (AsciiTable &, int, int); @@ -149,11 +152,11 @@ extern void write_line (const ConcatString &, ofstream *); //////////////////////////////////////////////////////////////////////// -extern double compute_ss_index(const ConcatString &, LineDataFile &, - STATAnalysisJob &, int &, int &); +extern void compute_ss_index(LineDataFile &, STATAnalysisJob &, + AggrSSIndexInfo &, int &, int &); extern void write_case_cols(const ConcatString &, AsciiTable &, - int &, int &); + int &, int &); //////////////////////////////////////////////////////////////////////// From 545c56b0a932b6da2bc87e13f26afc953253d68a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 23 Aug 2021 15:26:52 -0600 Subject: [PATCH 14/60] Per #1788 fix typo. --- met/src/basic/vx_util/stat_column_defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/src/basic/vx_util/stat_column_defs.h b/met/src/basic/vx_util/stat_column_defs.h index c6afa8d326..0b7302f8f3 100644 --- a/met/src/basic/vx_util/stat_column_defs.h +++ b/met/src/basic/vx_util/stat_column_defs.h @@ -1,4 +1,4 @@ -/// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* // ** Copyright UCAR (c) 1992 - 2021 // ** University Corporation for Atmospheric Research (UCAR) // ** National Center for Atmospheric Research (NCAR) From 32902530c3eea258c8365bc378b128c192ddc9d8 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 23 Aug 2021 15:43:49 -0600 Subject: [PATCH 15/60] Per #1788, remove GO Index defintion from the code comments. That detail belongs elsewhere. --- met/src/tools/core/stat_analysis/stat_analysis_job.cc | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index a094ff5554..2dccf28f4e 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -4003,17 +4003,6 @@ void write_job_ss_index(STATAnalysisJob &job, // Index = sqrt(1/(1-SS[avg])) // Where SS[avg] is the weighted average of skill scores. // -// GO Index definition: -// For lead times of 12, 24, 36, and 48 hours, it contains RMSE for: -// - Wind Speed at the surface(b), 850(a), 400(a), 250(a) mb -// - Dewpoint Temperature at the surface(b), 850(b), 700(b), 400(b) mB -// - Temperature at the surface(b), 400(a) mB -// - Height at 400(a) mB -// - Sea Level Pressure(b) -// Where (a) means weights of 4, 3, 2, 1 for the lead times, and -// (b) means weights of 8, 6, 4, 2 for the lead times. -// The RMSE values are dervied from the partial sums in the SL1L2 lines. -// //////////////////////////////////////////////////////////////////////// void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, From f2a17ee9a6c84e8b730099fcbdacff968c11a6f4 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 23 Aug 2021 16:20:25 -0600 Subject: [PATCH 16/60] Per #1788, since coding up the GO Index several CNT and CTC columns have been added. Adding them here to let them be included as part of the Skill Score index computations. --- .../core/stat_analysis/stat_analysis_job.cc | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index 2dccf28f4e..5508f1df15 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -4612,10 +4612,10 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, // // Extract the statistic to be used in defining the skill score. // Continuous (only stats derived from SL1L2 lines): - // PR_CORR, ME, ESTDEV, MBIAS, MSE, BCRMSE, RMSE + // PR_CORR, ME, ESTDEV, MBIAS, MAE, MSE, BCRMSE, RMSE, ME2, MSESS, SI // Categorical: // BASER, FMEAN, ACC, FBIAS, PODY, PODN, POFD, FAR, CSI, GSS, - // HK, HSS, ODDS + // HK, HSS, ODDS, LODDS, ORSS, EDS, SEDS, EDI, SEDI, BAGSS // fcst_stat = ref_stat = bad_data_double; @@ -4636,6 +4636,10 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, fcst_stat = fcst_cnt.mbias.v; ref_stat = ref_cnt.mbias.v; } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "MAE") == 0) { + fcst_stat = fcst_cnt.mae.v; + ref_stat = ref_cnt.mae.v; + } else if(strcasecmp(fcst_job[i].column[0].c_str(), "MSE") == 0) { fcst_stat = fcst_cnt.mse.v; ref_stat = ref_cnt.mse.v; @@ -4648,6 +4652,18 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, fcst_stat = fcst_cnt.rmse.v; ref_stat = ref_cnt.rmse.v; } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "ME2") == 0) { + fcst_stat = fcst_cnt.me2.v; + ref_stat = ref_cnt.me2.v; + } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "MSESS") == 0) { + fcst_stat = fcst_cnt.msess.v; + ref_stat = ref_cnt.msess.v; + } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "SI") == 0) { + fcst_stat = fcst_cnt.si.v; + ref_stat = ref_cnt.si.v; + } else if(strcasecmp(fcst_job[i].column[0].c_str(), "BASER") == 0) { fcst_stat = fcst_cts[i].baser.v; ref_stat = ref_cts[i].baser.v; @@ -4700,6 +4716,34 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, fcst_stat = fcst_cts[i].odds.v; ref_stat = ref_cts[i].odds.v; } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "LODDS") == 0) { + fcst_stat = fcst_cts[i].lodds.v; + ref_stat = ref_cts[i].lodds.v; + } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "ORSS") == 0) { + fcst_stat = fcst_cts[i].orss.v; + ref_stat = ref_cts[i].orss.v; + } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "EDS") == 0) { + fcst_stat = fcst_cts[i].eds.v; + ref_stat = ref_cts[i].eds.v; + } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "SEDS") == 0) { + fcst_stat = fcst_cts[i].seds.v; + ref_stat = ref_cts[i].seds.v; + } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "EDI") == 0) { + fcst_stat = fcst_cts[i].edi.v; + ref_stat = ref_cts[i].edi.v; + } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "SEDI") == 0) { + fcst_stat = fcst_cts[i].sedi.v; + ref_stat = ref_cts[i].sedi.v; + } + else if(strcasecmp(fcst_job[i].column[0].c_str(), "BAGSS") == 0) { + fcst_stat = fcst_cts[i].bagss.v; + ref_stat = ref_cts[i].bagss.v; + } // // Check for conditions when a skill score cannot be computed. From d6154e57320bcbedf53ceedecebef7ec2877da90 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 23 Aug 2021 17:00:22 -0600 Subject: [PATCH 17/60] Per #1788, update check_hdr_str() to print the name of the problematic header columns. --- met/src/basic/vx_util/ascii_table.cc | 20 +++++++------ met/src/basic/vx_util/ascii_table.h | 2 +- met/src/libcode/vx_shapedata/engine.cc | 28 ++++++++++++------- .../libcode/vx_stat_out/stat_hdr_columns.cc | 24 ++++++++-------- .../other/mode_time_domain/mtd_config_info.cc | 16 +++++------ 5 files changed, 50 insertions(+), 40 deletions(-) diff --git a/met/src/basic/vx_util/ascii_table.cc b/met/src/basic/vx_util/ascii_table.cc index 910e5165d9..3bc688a2da 100644 --- a/met/src/basic/vx_util/ascii_table.cc +++ b/met/src/basic/vx_util/ascii_table.cc @@ -1692,27 +1692,29 @@ void justify_met_at(AsciiTable &at, const int n_hdr_cols) { //////////////////////////////////////////////////////////////////////// -ConcatString check_hdr_str(const ConcatString s, bool space_to_underscore) { - ConcatString s_tmp = s; +ConcatString check_hdr_str(const ConcatString &col_name, + const ConcatString &col_value, + bool space_to_underscore) { + ConcatString cs_tmp = col_value; - if(space_to_underscore) s_tmp.replace(" ", "_", false); + if(space_to_underscore) cs_tmp.replace(" ", "_", false); // Check for empty string - if(s_tmp.length() == 0) { + if(cs_tmp.length() == 0) { mlog << Warning << "\ncheck_hdr_str() -> " - << "null string!\n\n"; + << "output header column " << to_upper(col_name) << " is empty!\n\n"; return(na_string); } // Check for embedded whitespace - if(check_reg_exp(ws_reg_exp, s_tmp.c_str())) { + if(check_reg_exp(ws_reg_exp, cs_tmp.c_str())) { mlog << Error << "\ncheck_hdr_str() -> " - << "output header column value (\"" << s_tmp - << "\") should contain no embedded whitespace!\n\n"; + << "output header column " << to_upper(col_name) << " value (\"" + << cs_tmp << "\") should contain no embedded whitespace!\n\n"; exit(1); } - return(s_tmp); + return(cs_tmp); } diff --git a/met/src/basic/vx_util/ascii_table.h b/met/src/basic/vx_util/ascii_table.h index 4e0ed53804..e75f64876c 100644 --- a/met/src/basic/vx_util/ascii_table.h +++ b/met/src/basic/vx_util/ascii_table.h @@ -366,7 +366,7 @@ extern void copy_ascii_table_row(const AsciiTable &at_from, const int r_from, As extern void justify_met_at(AsciiTable &at, const int n_hdr_cols); -extern ConcatString check_hdr_str(const ConcatString, +extern ConcatString check_hdr_str(const ConcatString &, const ConcatString &, bool space_to_underscore = false); diff --git a/met/src/libcode/vx_shapedata/engine.cc b/met/src/libcode/vx_shapedata/engine.cc index ff0b98e393..cbb8974c96 100644 --- a/met/src/libcode/vx_shapedata/engine.cc +++ b/met/src/libcode/vx_shapedata/engine.cc @@ -2773,10 +2773,11 @@ void write_header_columns(ModeFuzzyEngine & eng, const Grid & grid, AsciiTable & // Version at.set_entry(row, c++, - (string)met_version); + (string) met_version); // Model Name - s = check_hdr_str(eng.conf_info.model); + s = check_hdr_str(conf_key_model, + eng.conf_info.model); at.set_entry(row, c++, s.text()); // N_valid @@ -2787,7 +2788,7 @@ void write_header_columns(ModeFuzzyEngine & eng, const Grid & grid, AsciiTable & // Description - s = check_hdr_str(eng.conf_info.desc); + s = check_hdr_str(conf_key_desc, eng.conf_info.desc); at.set_entry(row, c++, s.text()); // Forecast lead time @@ -2831,31 +2832,38 @@ void write_header_columns(ModeFuzzyEngine & eng, const Grid & grid, AsciiTable & eng.conf_info.obs_conv_thresh.get_str()); // Forecast Variable Name - s = check_hdr_str(eng.conf_info.fcst_info->name_attr()); + s = check_hdr_str(conf_key_fcst_var, + eng.conf_info.fcst_info->name_attr()); at.set_entry(row, c++, s.text()); // Forecast Variable Units - s = check_hdr_str(eng.conf_info.fcst_info->units_attr(), true); + s = check_hdr_str(conf_key_fcst_units, + eng.conf_info.fcst_info->units_attr(), true); at.set_entry(row, c++, s.text()); // Forecast Variable Level - s = check_hdr_str(eng.conf_info.fcst_info->level_attr(), true); + s = check_hdr_str(conf_key_fcst_lev, + eng.conf_info.fcst_info->level_attr(), true); at.set_entry(row, c++, s.text()); // Observation Variable Name - s = check_hdr_str(eng.conf_info.obs_info->name_attr()); + s = check_hdr_str(conf_key_obs_var, + eng.conf_info.obs_info->name_attr()); at.set_entry(row, c++, s.text()); // Observation Variable Units - s = check_hdr_str(eng.conf_info.obs_info->units_attr(), true); + s = check_hdr_str(conf_key_obs_units, + eng.conf_info.obs_info->units_attr(), true); at.set_entry(row, c++, s.text()); // Observation Variable Level - s = check_hdr_str(eng.conf_info.obs_info->level_attr(), true); + s = check_hdr_str(conf_key_obs_lev, + eng.conf_info.obs_info->level_attr(), true); at.set_entry(row, c++, s.text()); // Observation type - s = check_hdr_str(eng.conf_info.obtype); + s = check_hdr_str(conf_key_obtype, + eng.conf_info.obtype); at.set_entry(row, c++, s.text()); return; diff --git a/met/src/libcode/vx_stat_out/stat_hdr_columns.cc b/met/src/libcode/vx_stat_out/stat_hdr_columns.cc index 2ab360b744..23ce8fe8f0 100644 --- a/met/src/libcode/vx_stat_out/stat_hdr_columns.cc +++ b/met/src/libcode/vx_stat_out/stat_hdr_columns.cc @@ -99,14 +99,14 @@ void StatHdrColumns::clear() { //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_model(const char *s) { - model = check_hdr_str((string)s); + model = check_hdr_str(conf_key_model, (string) s); return; } //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_desc(const char *s) { - desc = check_hdr_str((string)s); + desc = check_hdr_str(conf_key_desc, (string) s); return; } @@ -161,56 +161,56 @@ void StatHdrColumns::set_obs_valid_end(const unixtime ut) { //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_fcst_var(const ConcatString s) { - fcst_var = check_hdr_str(s); + fcst_var = check_hdr_str(conf_key_fcst_var, s); return; } //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_fcst_units(const ConcatString s) { - fcst_units = check_hdr_str(s, true); + fcst_units = check_hdr_str(conf_key_fcst_units, s, true); return; } //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_fcst_lev(const char *s) { - fcst_lev = check_hdr_str((string)s, true); + fcst_lev = check_hdr_str(conf_key_fcst_lev, (string) s, true); return; } //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_obs_var(const ConcatString s) { - obs_var = check_hdr_str(s); + obs_var = check_hdr_str(conf_key_obs_var, s); return; } //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_obs_units(const ConcatString s) { - obs_units = check_hdr_str(s, true); + obs_units = check_hdr_str(conf_key_obs_units, s, true); return; } //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_obs_lev(const char *s) { - obs_lev = check_hdr_str((string)s, true); + obs_lev = check_hdr_str(conf_key_obs_lev, (string) s, true); return; } //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_obtype(const char *s) { - obtype = check_hdr_str((string)s); + obtype = check_hdr_str(conf_key_obtype, (string) s); return; } //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_mask(const char *s) { - mask = check_hdr_str((string)s); + mask = check_hdr_str(conf_key_vx_mask, (string) s); return; } @@ -234,7 +234,7 @@ void StatHdrColumns::set_interp_mthd(ConcatString s, GridTemplateFactory gtf; mthd << '_' << gtf.enum2String(shape).c_str(); } - interp_mthd = check_hdr_str(mthd); + interp_mthd = check_hdr_str(conf_key_interp_mthd, mthd); return; } @@ -269,7 +269,7 @@ void StatHdrColumns::set_interp_pnts(const int n) { //////////////////////////////////////////////////////////////////////// void StatHdrColumns::set_line_type(const char *s) { - line_type = check_hdr_str((string)s); + line_type = check_hdr_str(conf_key_line_type, (string) s); return; } diff --git a/met/src/tools/other/mode_time_domain/mtd_config_info.cc b/met/src/tools/other/mode_time_domain/mtd_config_info.cc index 8f700c0b85..baa789d11c 100644 --- a/met/src/tools/other/mode_time_domain/mtd_config_info.cc +++ b/met/src/tools/other/mode_time_domain/mtd_config_info.cc @@ -697,13 +697,13 @@ table.set_entry(row, c++, (string)met_version); // model -s = check_hdr_str(model); +s = check_hdr_str(conf_key_model, model); table.set_entry(row, c++, s.text()); // description -s = check_hdr_str(desc); +s = check_hdr_str(conf_key_desc, desc); table.set_entry(row, c++, s.text()); @@ -765,37 +765,37 @@ table.set_entry(row, c++, s.text()); // fcst var -s = check_hdr_str(fcst_info->name_attr()); +s = check_hdr_str(conf_key_fcst_var, fcst_info->name_attr()); table.set_entry(row, c++, s.text()); // fcst units -s = check_hdr_str(fcst_info->units_attr(), true); +s = check_hdr_str(conf_key_fcst_units, fcst_info->units_attr(), true); table.set_entry(row, c++, s.text()); // fcst level -s = check_hdr_str(fcst_info->level_attr(), true); +s = check_hdr_str(conf_key_fcst_lev, fcst_info->level_attr(), true); table.set_entry(row, c++, s.text()); // obs var -s = check_hdr_str(obs_info->name_attr()); +s = check_hdr_str(conf_key_obs_var, obs_info->name_attr()); table.set_entry(row, c++, s.text()); // obs units -s = check_hdr_str(obs_info->units_attr(), true); +s = check_hdr_str(conf_key_obs_units, obs_info->units_attr(), true); table.set_entry(row, c++, s.text()); // obs level -s = check_hdr_str(obs_info->level_attr(), true); +s = check_hdr_str(conf_key_obs_lev, obs_info->level_attr(), true); table.set_entry(row, c++, s.text()); From f1d12b745da8410ce51fd7933c571dac1aaf40d9 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 24 Aug 2021 11:41:59 -0600 Subject: [PATCH 18/60] Per #1788, add a set(one) member function to multiple array classes. This calls clear or erase on the array prior to adding a single element. Add this for StringArray, NumArray, and ThreshArray, and CRCArray. --- met/src/basic/vx_log/string_array.cc | 19 +++++++++++--- met/src/basic/vx_log/string_array.h | 2 ++ met/src/basic/vx_util/crc_array.h | 20 ++++++++++++++ met/src/basic/vx_util/num_array.cc | 38 +++++++++++++++++++++++++-- met/src/basic/vx_util/num_array.h | 2 ++ met/src/basic/vx_util/thresh_array.cc | 10 +++++++ met/src/basic/vx_util/thresh_array.h | 2 ++ 7 files changed, 88 insertions(+), 5 deletions(-) diff --git a/met/src/basic/vx_log/string_array.cc b/met/src/basic/vx_log/string_array.cc index 5894d80f2e..21419fbe6c 100644 --- a/met/src/basic/vx_log/string_array.cc +++ b/met/src/basic/vx_log/string_array.cc @@ -134,9 +134,6 @@ IgnoreCase = 0; clear(); - - - return; } @@ -308,6 +305,22 @@ void StringArray::add_css(const std::string text) //////////////////////////////////////////////////////////////////////// +void StringArray::set(const std::string text) + +{ + +s.clear(); + +s.push_back(text); + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + void StringArray::set(int i, const std::string text) { diff --git a/met/src/basic/vx_log/string_array.h b/met/src/basic/vx_log/string_array.h index d940ef3958..6563f68e2e 100644 --- a/met/src/basic/vx_log/string_array.h +++ b/met/src/basic/vx_log/string_array.h @@ -66,6 +66,8 @@ class StringArray { void add_css(const std::string); + void set(const std::string); + void set(int i, const std::string); void insert(int i, const char *); diff --git a/met/src/basic/vx_util/crc_array.h b/met/src/basic/vx_util/crc_array.h index 4fa5a99e6c..d79886b62c 100644 --- a/met/src/basic/vx_util/crc_array.h +++ b/met/src/basic/vx_util/crc_array.h @@ -130,6 +130,7 @@ class CRC_Array { void add(const CRC_Array &); void add_css_sec(const char *); + void set(const T & val); void set(int ix, const T & val); void sort_increasing(); @@ -394,6 +395,24 @@ return; //////////////////////////////////////////////////////////////////////// +template + +void CRC_Array::set(const T & elem) + +{ + +clear(); + +add(elem); + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + template void CRC_Array::set(int ix, const T & elem) @@ -410,6 +429,7 @@ void CRC_Array::set(int ix, const T & elem) exit ( 1 ); } + e[ix] = elem; } diff --git a/met/src/basic/vx_util/num_array.cc b/met/src/basic/vx_util/num_array.cc index 9493b14fb2..3a6b10e5a1 100644 --- a/met/src/basic/vx_util/num_array.cc +++ b/met/src/basic/vx_util/num_array.cc @@ -495,14 +495,48 @@ return; //////////////////////////////////////////////////////////////////////// +void NumArray::set(int k) + +{ + +set((double) k); + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + +void NumArray::set(double d) + +{ + +erase(); + +add(d); + + // + // an array of length one is sorted + // + +Sorted = true; + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + void NumArray::set(int i, int k) { set(i, (double) k); -Sorted = false; - return; } diff --git a/met/src/basic/vx_util/num_array.h b/met/src/basic/vx_util/num_array.h index 8c24299a42..eaea0f7990 100644 --- a/met/src/basic/vx_util/num_array.h +++ b/met/src/basic/vx_util/num_array.h @@ -79,6 +79,8 @@ class NumArray { void add_css(const char *); void add_css_sec(const char *); + void set(int); + void set(double); void set(int, int); void set(int, double); diff --git a/met/src/basic/vx_util/thresh_array.cc b/met/src/basic/vx_util/thresh_array.cc index 50358a5d83..faafb0d98e 100644 --- a/met/src/basic/vx_util/thresh_array.cc +++ b/met/src/basic/vx_util/thresh_array.cc @@ -256,6 +256,16 @@ void ThreshArray::add_css(const char *text) { //////////////////////////////////////////////////////////////////////// +void ThreshArray::set(const SingleThresh &st) { + + clear(); + + add(st); + + return; +} +//////////////////////////////////////////////////////////////////////// + void ThreshArray::parse_thresh_str(const char *thresh_str) { char *line = (char *) 0; char *c = (char *) 0; diff --git a/met/src/basic/vx_util/thresh_array.h b/met/src/basic/vx_util/thresh_array.h index d27dc087c9..4c205dd3a8 100644 --- a/met/src/basic/vx_util/thresh_array.h +++ b/met/src/basic/vx_util/thresh_array.h @@ -57,6 +57,8 @@ class ThreshArray { void add(const ThreshArray &); void add_css(const char *); + void set(const SingleThresh &); + void parse_thresh_str(const char *); int n_elements() const; From 753a07be65e0c9ab5561483a7d24a1dd476c130b Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 24 Aug 2021 11:44:16 -0600 Subject: [PATCH 19/60] Per #1788, update the compute_ss_index() function in stat_analysis to make the code more concise by calling the newly adding singular set functions for arrays. --- .../core/stat_analysis/stat_analysis_job.cc | 234 ++++++------------ 1 file changed, 71 insertions(+), 163 deletions(-) diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index 5508f1df15..13a349a000 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -3961,9 +3961,12 @@ void write_job_ss_index(STATAnalysisJob &job, // Write the output STAT header columns // ConcatString cur_case(""); + shc = ssidx_info.hdr.get_shc(cur_case, job.by_column, job.hdr_name, job.hdr_value, stat_ssidx); + // JHG, set some of the stat header columns here! + // // Initialize // @@ -4244,7 +4247,7 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, AggrSSIndexInfo &ssidx_info, int &n_in, int &n_out) { STATLine line; - SL1L2Info si; + SL1L2Info sl1l2; TTContingencyTable ct; CNTInfo fcst_cnt, ref_cnt; bool keep; @@ -4303,12 +4306,12 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, // Define arrays of objects to store the partial sums or contingency // table counts for each term in the Skill Score Index. // - SL1L2Info *fcst_si = (SL1L2Info *) 0, *ref_si = (SL1L2Info *) 0; - CTSInfo *fcst_cts = (CTSInfo *) 0, *ref_cts = (CTSInfo *) 0; - fcst_si = new SL1L2Info [n_term]; - ref_si = new SL1L2Info [n_term]; - fcst_cts = new CTSInfo [n_term]; - ref_cts = new CTSInfo [n_term]; + SL1L2Info *fcst_sl1l2 = (SL1L2Info *) 0, *ref_sl1l2 = (SL1L2Info *) 0; + CTSInfo *fcst_cts = (CTSInfo *) 0, *ref_cts = (CTSInfo *) 0; + fcst_sl1l2 = new SL1L2Info [n_term]; + ref_sl1l2 = new SL1L2Info [n_term]; + fcst_cts = new CTSInfo [n_term]; + ref_cts = new CTSInfo [n_term]; // // Define array of line types to be aggregated for each term in the @@ -4322,7 +4325,8 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, // NumArray n_fcst_lines, n_ref_lines; - mlog << Debug(3) << "Forecast Model = " << job.model[0] << "\n" + mlog << Debug(3) + << "Forecast Model = " << job.model[0] << "\n" << "Reference Model = " << job.model[1] << "\n"; // @@ -4342,122 +4346,25 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, fcst_job[i] = job; // - // model - // - fcst_job[i].model.clear(); - fcst_job[i].model.add(job.model[0]); - - // - // fcst_lead - // - if(job.fcst_lead.n() == n_term) { - fcst_job[i].fcst_lead.clear(); - fcst_job[i].fcst_lead.add(job.fcst_lead[i]); - } - - // - // obs_lead - // - if(job.obs_lead.n() == n_term) { - fcst_job[i].obs_lead.clear(); - fcst_job[i].obs_lead.add(job.obs_lead[i]); - } - - // - // fcst_init_hour - // - if(job.fcst_init_hour.n() == n_term) { - fcst_job[i].fcst_init_hour.clear(); - fcst_job[i].fcst_init_hour.add(job.fcst_init_hour[i]); - } - - // - // obs_init_hour - // - if(job.obs_init_hour.n() == n_term) { - fcst_job[i].obs_init_hour.clear(); - fcst_job[i].obs_init_hour.add(job.obs_init_hour[i]); - } - - // - // fcst_var - // - if(job.fcst_var.n() == n_term) { - fcst_job[i].fcst_var.clear(); - fcst_job[i].fcst_var.add(job.fcst_var[i]); - } - - // - // obs_var - // - if(job.obs_var.n() == n_term) { - fcst_job[i].obs_var.clear(); - fcst_job[i].obs_var.add(job.obs_var[i]); - } - - // - // fcst_lev + // Set filtering options // - if(job.fcst_lev.n() == n_term) { - fcst_job[i].fcst_lev.clear(); - fcst_job[i].fcst_lev.add(job.fcst_lev[i]); - } - - // - // obs_lev - // - if(job.obs_lev.n() == n_term) { - fcst_job[i].obs_lev.clear(); - fcst_job[i].obs_lev.add(job.obs_lev[i]); - } - - // - // obtype - // - if(job.obtype.n() == n_term) { - fcst_job[i].obtype.clear(); - fcst_job[i].obtype.add(job.obtype[i]); - } - - // - // vx_mask - // - if(job.vx_mask.n() == n_term) { - fcst_job[i].vx_mask.clear(); - fcst_job[i].vx_mask.add(job.vx_mask[i]); - } - - // - // interp_mthd - // - if(job.interp_mthd.n() == n_term) { - fcst_job[i].interp_mthd.clear(); - fcst_job[i].interp_mthd.add(job.interp_mthd[i]); - } - - // - // interp_pnts - // - if(job.interp_pnts.n() == n_term) { - fcst_job[i].interp_pnts.clear(); - fcst_job[i].interp_pnts.add(job.interp_pnts[i]); - } - - // - // fcst_thresh - // - if(job.fcst_thresh.n() == n_term) { - fcst_job[i].fcst_thresh.clear(); - fcst_job[i].fcst_thresh.add(job.fcst_thresh[i]); - } - - // - // obs_thresh - // - if(job.obs_thresh.n() == n_term) { - fcst_job[i].obs_thresh.clear(); - fcst_job[i].obs_thresh.add(job.obs_thresh[i]); - } + fcst_job[i].model.set(job.model[0]); + if(job.fcst_lead.n() == n_term) fcst_job[i].fcst_lead.set(job.fcst_lead[i]); + if(job.obs_lead.n() == n_term) fcst_job[i].obs_lead.set(job.obs_lead[i]); + if(job.fcst_init_hour.n() == n_term) fcst_job[i].fcst_init_hour.set(job.fcst_init_hour[i]); + if(job.obs_init_hour.n() == n_term) fcst_job[i].obs_init_hour.set(job.obs_init_hour[i]); + if(job.fcst_var.n() == n_term) fcst_job[i].fcst_var.set(job.fcst_var[i]); + if(job.obs_var.n() == n_term) fcst_job[i].obs_var.set(job.obs_var[i]); + if(job.fcst_lev.n() == n_term) fcst_job[i].fcst_lev.add(job.fcst_lev[i]); + if(job.obs_lev.n() == n_term) fcst_job[i].obs_lev.set(job.obs_lev[i]); + if(job.obtype.n() == n_term) fcst_job[i].obtype.set(job.obtype[i]); + if(job.vx_mask.n() == n_term) fcst_job[i].vx_mask.set(job.vx_mask[i]); + if(job.interp_mthd.n() == n_term) fcst_job[i].interp_mthd.set(job.interp_mthd[i]); + if(job.interp_pnts.n() == n_term) fcst_job[i].interp_pnts.set(job.interp_pnts[i]); + if(job.fcst_thresh.n() == n_term) fcst_job[i].fcst_thresh.set(job.fcst_thresh[i]); + if(job.obs_thresh.n() == n_term) fcst_job[i].obs_thresh.set(job.obs_thresh[i]); + if(job.column.n() == n_term) fcst_job[i].column.set(job.column[i]); + if(job.weight.n() == n_term) fcst_job[i].weight.set(job.weight[i]); // // line_type @@ -4468,13 +4375,13 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, job_lt[i] = string_to_statlinetype(job.line_type[i].c_str()); if(job_lt[i] != stat_sl1l2 && job_lt[i] != stat_ctc) { - if ( fcst_job ) { delete [] fcst_job; fcst_job = 0; } - if ( ref_job ) { delete [] ref_job; ref_job = 0; } - if ( fcst_si ) { delete [] fcst_si; fcst_si = 0; } - if ( ref_si ) { delete [] ref_si; ref_si = 0; } - if ( fcst_cts ) { delete [] fcst_cts; fcst_cts = 0; } - if ( ref_cts ) { delete [] ref_cts; ref_cts = 0; } - if ( job_lt ) { delete [] job_lt; job_lt = 0; } + if(fcst_job ) { delete [] fcst_job; fcst_job = 0; } + if(ref_job ) { delete [] ref_job; ref_job = 0; } + if(fcst_sl1l2) { delete [] fcst_sl1l2; fcst_sl1l2 = 0; } + if(ref_sl1l2 ) { delete [] ref_sl1l2; ref_sl1l2 = 0; } + if(fcst_cts ) { delete [] fcst_cts; fcst_cts = 0; } + if(ref_cts ) { delete [] ref_cts; ref_cts = 0; } + if(job_lt ) { delete [] job_lt; job_lt = 0; } mlog << Error << "\ncompute_ss_index() -> " << "a Skill Score Index can only be computed using " << "statistics derived from SL1L2 or CTC line types." @@ -4483,28 +4390,12 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, } } - // - // column - // - if(job.column.n() == n_term) { - fcst_job[i].column.clear(); - fcst_job[i].column.add(job.column[i]); - } - - // - // weight - // - if(job.weight.n() == n_term) { - fcst_job[i].weight.clear(); - fcst_job[i].weight.add(job.weight[i]); - } - // // Set the reference model job identical to the forecast model // job but with a different model name. // ref_job[i] = fcst_job[i]; - ref_job[i].model.set(0, job.model[1]); + ref_job[i].model.set(job.model[1]); } // end for i @@ -4531,10 +4422,15 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, keep = 1; n_fcst_lines.set(i, n_fcst_lines[i]+1); + // + // Keep track of the unique header columne entries + // + ssidx_info.hdr.add(line); + if(job_lt[i] == stat_sl1l2) { - si.clear(); - parse_sl1l2_line(line, si); - fcst_si[i] += si; + sl1l2.clear(); + parse_sl1l2_line(line, sl1l2); + fcst_sl1l2[i] += sl1l2; } else if(job_lt[i] == stat_ctc) { ct.zero_out(); @@ -4557,10 +4453,15 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, keep = 1; n_ref_lines.set(i, n_ref_lines[i]+1); + // + // Keep track of the unique header columne entries + // + ssidx_info.hdr.add(line); + if(job_lt[i] == stat_sl1l2) { - si.clear(); - parse_sl1l2_line(line, si); - ref_si[i] += si; + sl1l2.clear(); + parse_sl1l2_line(line, sl1l2); + ref_sl1l2[i] += sl1l2; } else if(job_lt[i]== stat_ctc) { ct.zero_out(); @@ -4597,9 +4498,9 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, // if(job_lt[i] == stat_sl1l2) { fcst_cnt.clear(); - compute_cntinfo(fcst_si[i], 0, fcst_cnt); + compute_cntinfo(fcst_sl1l2[i], 0, fcst_cnt); ref_cnt.clear(); - compute_cntinfo(ref_si[i], 0, ref_cnt); + compute_cntinfo(ref_sl1l2[i], 0, ref_cnt); } // // Compute categorical stats for the current term @@ -4612,12 +4513,19 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, // // Extract the statistic to be used in defining the skill score. // Continuous (only stats derived from SL1L2 lines): - // PR_CORR, ME, ESTDEV, MBIAS, MAE, MSE, BCRMSE, RMSE, ME2, MSESS, SI + // PR_CORR, ME, ESTDEV, MBIAS, MAE, MSE, BCRMSE, RMSE, ME2, + // MSESS, SI // Categorical: // BASER, FMEAN, ACC, FBIAS, PODY, PODN, POFD, FAR, CSI, GSS, // HK, HSS, ODDS, LODDS, ORSS, EDS, SEDS, EDI, SEDI, BAGSS // + +// JHG, can we move this code and the series_analysis code to get stats by name to a library? +// fcst_stat = fcst_cnt.get_stat(fcst_job[i].column[0]); +// ref_stat = ref_cnt.get_stat(fcst_job[i].column[0]); + + fcst_stat = ref_stat = bad_data_double; if(strcasecmp(fcst_job[i].column[0].c_str(), "PR_CORR") == 0) { @@ -4820,13 +4728,13 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, // // Clean up allocated memory. // - if(fcst_job) { delete [] fcst_job; fcst_job = (STATAnalysisJob *) 0; } - if(ref_job) { delete [] ref_job; ref_job = (STATAnalysisJob *) 0; } - if(fcst_si) { delete [] fcst_si; fcst_si = (SL1L2Info *) 0; } - if(ref_si) { delete [] ref_si; ref_si = (SL1L2Info *) 0; } - if(fcst_cts) { delete [] fcst_cts; fcst_cts = (CTSInfo *) 0; } - if(ref_cts) { delete [] ref_cts; ref_cts = (CTSInfo *) 0; } - if(job_lt) { delete [] job_lt; job_lt = (STATLineType *) 0; } + if(fcst_job) { delete [] fcst_job; fcst_job = (STATAnalysisJob *) 0; } + if(ref_job) { delete [] ref_job; ref_job = (STATAnalysisJob *) 0; } + if(fcst_sl1l2) { delete [] fcst_sl1l2; fcst_sl1l2 = (SL1L2Info *) 0; } + if(ref_sl1l2) { delete [] ref_sl1l2; ref_sl1l2 = (SL1L2Info *) 0; } + if(fcst_cts) { delete [] fcst_cts; fcst_cts = (CTSInfo *) 0; } + if(ref_cts) { delete [] ref_cts; ref_cts = (CTSInfo *) 0; } + if(job_lt) { delete [] job_lt; job_lt = (STATLineType *) 0; } return; } From 0a48249809c9170248bf2ea1456729624b540dd6 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 24 Aug 2021 12:26:20 -0600 Subject: [PATCH 20/60] Per #1788, add FCST_MODEL and REF_MODEL output columns to the SSIDX line type. --- .../table_files/met_header_columns_V10.1.txt | 2 +- met/src/basic/vx_util/stat_column_defs.h | 3 ++- met/src/libcode/vx_stat_out/stat_columns.cc | 17 +++++++------ met/src/libcode/vx_statistics/met_stats.h | 1 + .../core/stat_analysis/stat_analysis_job.cc | 25 ++++++++++++++----- test/hdr/met_10_1.hdr | 2 +- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/met/data/table_files/met_header_columns_V10.1.txt b/met/data/table_files/met_header_columns_V10.1.txt index ca4c40bed5..5f03639559 100644 --- a/met/data/table_files/met_header_columns_V10.1.txt +++ b/met/data/table_files/met_header_columns_V10.1.txt @@ -29,7 +29,7 @@ V10.1 : STAT : VAL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID V10.1 : STAT : VL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL UFBAR VFBAR UOBAR VOBAR UVFOBAR UVFFBAR UVOOBAR F_SPEED_BAR O_SPEED_BAR V10.1 : STAT : VCNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FBAR FBAR_BCL FBAR_BCU OBAR OBAR_BCL OBAR_BCU FS_RMS FS_RMS_BCL FS_RMS_BCU OS_RMS OS_RMS_BCL OS_RMS_BCU MSVE MSVE_BCL MSVE_BCU RMSVE RMSVE_BCL RMSVE_BCU FSTDEV FSTDEV_BCL FSTDEV_BCU OSTDEV OSTDEV_BCL OSTDEV_BCU FDIR FDIR_BCL FDIR_BCU ODIR ODIR_BCL ODIR_BCU FBAR_SPEED FBAR_SPEED_BCL FBAR_SPEED_BCU OBAR_SPEED OBAR_SPEED_BCL OBAR_SPEED_BCU VDIFF_SPEED VDIFF_SPEED_BCL VDIFF_SPEED_BCU VDIFF_DIR VDIFF_DIR_BCL VDIFF_DIR_BCU SPEED_ERR SPEED_ERR_BCL SPEED_ERR_BCU SPEED_ABSERR SPEED_ABSERR_BCL SPEED_ABSERR_BCU DIR_ERR DIR_ERR_BCL DIR_ERR_BCU DIR_ABSERR DIR_ABSERR_BCL DIR_ABSERR_BCU V10.1 : STAT : GENMPR : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX STORM_ID AGEN_INIT AGEN_FHR AGEN_LAT AGEN_LON AGEN_DLAND BGEN_LAT BGEN_LON BGEN_DLAND GEN_DIST GEN_TDIFF INIT_TDIFF DEV_CAT OPS_CAT -V10.1 : STAT : SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE SS_INDEX +V10.1 : STAT : SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE FCST_MODEL REF_MODEL N_TERM N_VLD SS_INDEX V10.1 : MODE : OBJ : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE OBJECT_ID OBJECT_CAT CENTROID_X CENTROID_Y CENTROID_LAT CENTROID_LON AXIS_ANG LENGTH WIDTH AREA AREA_THRESH CURVATURE CURVATURE_X CURVATURE_Y COMPLEXITY INTENSITY_10 INTENSITY_25 INTENSITY_50 INTENSITY_75 INTENSITY_90 INTENSITY_USER INTENSITY_SUM CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST V10.1 : MODE : CTS : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS diff --git a/met/src/basic/vx_util/stat_column_defs.h b/met/src/basic/vx_util/stat_column_defs.h index 0b7302f8f3..f7075fdc0d 100644 --- a/met/src/basic/vx_util/stat_column_defs.h +++ b/met/src/basic/vx_util/stat_column_defs.h @@ -314,7 +314,8 @@ static const char * relp_columns [] = { }; static const char * ssidx_columns [] = { - "NAME", "N_TERM", "N_VLD", "SSINDEX" + "FCST_MODEL", "REF_MODEL", "N_TERM", + "N_VLD", "SS_INDEX" }; static const char * genmpr_columns [] = { diff --git a/met/src/libcode/vx_stat_out/stat_columns.cc b/met/src/libcode/vx_stat_out/stat_columns.cc index 04a4529ed2..67650e8cc1 100644 --- a/met/src/libcode/vx_stat_out/stat_columns.cc +++ b/met/src/libcode/vx_stat_out/stat_columns.cc @@ -4132,19 +4132,22 @@ void write_ssidx_cols(SSIndexInfo ssidx_info, // // Skill Score Index // Dump out the SSIDX line: - // NAME, N_TERM, N_VLD, - // SS_INDEX + // FCST_MODEL, REF_MODEL, N_TERM, + // N_VLD, SS_INDEX // - at.set_entry(r, c+0, // Skill score index name - ssidx_info.name); + at.set_entry(r, c+0, // Forecast model name + ssidx_info.fcst_model); - at.set_entry(r, c+1, // Number of terms + at.set_entry(r, c+1, // Reference model name + ssidx_info.ref_model); + + at.set_entry(r, c+2, // Number of terms ssidx_info.n_term); - at.set_entry(r, c+2, // Number of valid terms + at.set_entry(r, c+3, // Number of valid terms ssidx_info.n_vld); - at.set_entry(r, c+3, // Skill score index value + at.set_entry(r, c+4, // Skill score index value ssidx_info.ss_index); return; diff --git a/met/src/libcode/vx_statistics/met_stats.h b/met/src/libcode/vx_statistics/met_stats.h index 74fbd6bf7e..4a2a5ac780 100644 --- a/met/src/libcode/vx_statistics/met_stats.h +++ b/met/src/libcode/vx_statistics/met_stats.h @@ -672,6 +672,7 @@ inline double DMAPInfo::get_beta_value() const { return(beta_value); } struct SSIndexInfo { ConcatString name; + ConcatString fcst_model, ref_model; int n_term, n_vld; double ss_index; }; diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index 13a349a000..c3a83a635e 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -3965,7 +3965,16 @@ void write_job_ss_index(STATAnalysisJob &job, shc = ssidx_info.hdr.get_shc(cur_case, job.by_column, job.hdr_name, job.hdr_value, stat_ssidx); - // JHG, set some of the stat header columns here! + // + // Set FCST/OBS_VAR = skill score index type + // Set FCST/OBS_UNITS = FCST/OBS_LEV = NA + // + shc.set_fcst_var(ssidx_info.info.name); + shc.set_obs_var(ssidx_info.info.name); + shc.set_fcst_units(na_str); + shc.set_obs_units(na_str); + shc.set_fcst_lev(na_str); + shc.set_obs_lev(na_str); // // Initialize @@ -3982,7 +3991,7 @@ void write_job_ss_index(STATAnalysisJob &job, job.stat_row++, n_header_columns); } else { - at.set_entry(r, c++, "SSIDX:"); + at.set_entry(r, c++, ssidx_info.info.name); write_case_cols(cur_case, at, r, c); write_ssidx_cols(ssidx_info.info, at, r++, c); } @@ -4266,6 +4275,10 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, << "model followed by the reference model.\n\n"; throw(1); } + else { + ssidx_info.info.fcst_model = job.model[0]; + ssidx_info.info.ref_model = job.model[1]; + } // // Use the length of the fcst_var array to infer the number of terms. @@ -4326,8 +4339,8 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, NumArray n_fcst_lines, n_ref_lines; mlog << Debug(3) - << "Forecast Model = " << job.model[0] << "\n" - << "Reference Model = " << job.model[1] << "\n"; + << "Forecast Model = " << ssidx_info.info.fcst_model << "\n" + << "Reference Model = " << ssidx_info.info.ref_model << "\n"; // // Set up the job for each term in the index. @@ -4348,7 +4361,7 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, // // Set filtering options // - fcst_job[i].model.set(job.model[0]); + fcst_job[i].model.set(ssidx_info.info.fcst_model); if(job.fcst_lead.n() == n_term) fcst_job[i].fcst_lead.set(job.fcst_lead[i]); if(job.obs_lead.n() == n_term) fcst_job[i].obs_lead.set(job.obs_lead[i]); if(job.fcst_init_hour.n() == n_term) fcst_job[i].fcst_init_hour.set(job.fcst_init_hour[i]); @@ -4395,7 +4408,7 @@ void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, // job but with a different model name. // ref_job[i] = fcst_job[i]; - ref_job[i].model.set(job.model[1]); + ref_job[i].model.set(ssidx_info.info.ref_model); } // end for i diff --git a/test/hdr/met_10_1.hdr b/test/hdr/met_10_1.hdr index 9177d9dfb9..1c1866430e 100644 --- a/test/hdr/met_10_1.hdr +++ b/test/hdr/met_10_1.hdr @@ -29,7 +29,7 @@ VL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L VAL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL UFABAR VFABAR UOABAR VOABAR UVFOABAR UVFFABAR UVOOABAR VCNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FBAR FBAR_BCL FBAR_BCU OBAR OBAR_BCL OBAR_BCU FS_RMS FS_RMS_BCL FS_RMS_BCU OS_RMS OS_RMS_BCL OS_RMS_BCU MSVE MSVE_BCL MSVE_BCU RMSVE RMSVE_BCL RMSVE_BCU FSTDEV FSTDEV_BCL FSTDEV_BCU OSTDEV OSTDEV_BCL OSTDEV_BCU FDIR FDIR_BCL FDIR_BCU ODIR ODIR_BCL ODIR_BCU FBAR_SPEED FBAR_SPEED_BCL FBAR_SPEED_BCU OBAR_SPEED OBAR_SPEED_BCL OBAR_SPEED_BCU VDIFF_SPEED VDIFF_SPEED_BCL VDIFF_SPEED_BCU VDIFF_DIR VDIFF_DIR_BCL VDIFF_DIR_BCU SPEED_ERR SPEED_ERR_BCL SPEED_ERR_BCU SPEED_ABSERR SPEED_ABSERR_BCL SPEED_ABSERR_BCU DIR_ERR DIR_ERR_BCL DIR_ERR_BCU DIR_ABSERR DIR_ABSERR_BCL DIR_ABSERR_BCU GENMPR : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX STORM_ID AGEN_INIT AGEN_FHR AGEN_LAT AGEN_LON AGEN_DLAND BGEN_LAT BGEN_LON BGEN_DLAND GEN_DIST GEN_TDIFF INIT_TDIFF DEV_CAT OPS_CAT -SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE SS_INDEX +SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE FCST_MODEL REF_MODEL N_TERM N_VLD SS_INDEX MODE_SOA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES OBJECT_ID OBJECT_CAT CENTROID_X CENTROID_Y CENTROID_LAT CENTROID_LON AXIS_ANG LENGTH WIDTH AREA AREA_THRESH CURVATURE CURVATURE_X CURVATURE_Y COMPLEXITY INTENSITY_10 INTENSITY_25 INTENSITY_50 INTENSITY_75 INTENSITY_90 INTENSITY_50 INTENSITY_SUM MODE_POA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES OBJECT_ID OBJECT_CAT CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST MODE_CTS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS From 0734dd89d8985f2f373a60585c0433e2591a1246 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 12:21:53 -0600 Subject: [PATCH 21/60] Per #1788, add a new N_INIT column to the SSIDX line type to report the number of initialization times represented by this skill score. Also define the SSIDXData struct to store the skill score index job output. --- .../table_files/met_header_columns_V10.1.txt | 2 +- met/src/basic/vx_util/stat_column_defs.h | 4 +-- met/src/libcode/vx_stat_out/stat_columns.cc | 25 +++++++++++-------- met/src/libcode/vx_stat_out/stat_columns.h | 2 +- met/src/libcode/vx_statistics/met_stats.h | 18 ++++++++++--- test/hdr/met_10_1.hdr | 2 +- 6 files changed, 34 insertions(+), 19 deletions(-) diff --git a/met/data/table_files/met_header_columns_V10.1.txt b/met/data/table_files/met_header_columns_V10.1.txt index 5f03639559..c0218275ec 100644 --- a/met/data/table_files/met_header_columns_V10.1.txt +++ b/met/data/table_files/met_header_columns_V10.1.txt @@ -29,7 +29,7 @@ V10.1 : STAT : VAL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID V10.1 : STAT : VL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL UFBAR VFBAR UOBAR VOBAR UVFOBAR UVFFBAR UVOOBAR F_SPEED_BAR O_SPEED_BAR V10.1 : STAT : VCNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FBAR FBAR_BCL FBAR_BCU OBAR OBAR_BCL OBAR_BCU FS_RMS FS_RMS_BCL FS_RMS_BCU OS_RMS OS_RMS_BCL OS_RMS_BCU MSVE MSVE_BCL MSVE_BCU RMSVE RMSVE_BCL RMSVE_BCU FSTDEV FSTDEV_BCL FSTDEV_BCU OSTDEV OSTDEV_BCL OSTDEV_BCU FDIR FDIR_BCL FDIR_BCU ODIR ODIR_BCL ODIR_BCU FBAR_SPEED FBAR_SPEED_BCL FBAR_SPEED_BCU OBAR_SPEED OBAR_SPEED_BCL OBAR_SPEED_BCU VDIFF_SPEED VDIFF_SPEED_BCL VDIFF_SPEED_BCU VDIFF_DIR VDIFF_DIR_BCL VDIFF_DIR_BCU SPEED_ERR SPEED_ERR_BCL SPEED_ERR_BCU SPEED_ABSERR SPEED_ABSERR_BCL SPEED_ABSERR_BCU DIR_ERR DIR_ERR_BCL DIR_ERR_BCU DIR_ABSERR DIR_ABSERR_BCL DIR_ABSERR_BCU V10.1 : STAT : GENMPR : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX STORM_ID AGEN_INIT AGEN_FHR AGEN_LAT AGEN_LON AGEN_DLAND BGEN_LAT BGEN_LON BGEN_DLAND GEN_DIST GEN_TDIFF INIT_TDIFF DEV_CAT OPS_CAT -V10.1 : STAT : SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE FCST_MODEL REF_MODEL N_TERM N_VLD SS_INDEX +V10.1 : STAT : SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE FCST_MODEL REF_MODEL N_INIT N_TERM N_VLD SS_INDEX V10.1 : MODE : OBJ : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE OBJECT_ID OBJECT_CAT CENTROID_X CENTROID_Y CENTROID_LAT CENTROID_LON AXIS_ANG LENGTH WIDTH AREA AREA_THRESH CURVATURE CURVATURE_X CURVATURE_Y COMPLEXITY INTENSITY_10 INTENSITY_25 INTENSITY_50 INTENSITY_75 INTENSITY_90 INTENSITY_USER INTENSITY_SUM CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST V10.1 : MODE : CTS : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS diff --git a/met/src/basic/vx_util/stat_column_defs.h b/met/src/basic/vx_util/stat_column_defs.h index f7075fdc0d..b8ddc583bc 100644 --- a/met/src/basic/vx_util/stat_column_defs.h +++ b/met/src/basic/vx_util/stat_column_defs.h @@ -314,8 +314,8 @@ static const char * relp_columns [] = { }; static const char * ssidx_columns [] = { - "FCST_MODEL", "REF_MODEL", "N_TERM", - "N_VLD", "SS_INDEX" + "FCST_MODEL", "REF_MODEL", "N_INIT", + "N_TERM", "N_VLD", "SS_INDEX" }; static const char * genmpr_columns [] = { diff --git a/met/src/libcode/vx_stat_out/stat_columns.cc b/met/src/libcode/vx_stat_out/stat_columns.cc index 67650e8cc1..39707d1e67 100644 --- a/met/src/libcode/vx_stat_out/stat_columns.cc +++ b/met/src/libcode/vx_stat_out/stat_columns.cc @@ -4126,29 +4126,32 @@ void write_relp_cols(const PairDataEnsemble *pd_ptr, //////////////////////////////////////////////////////////////////////// -void write_ssidx_cols(SSIndexInfo ssidx_info, +void write_ssidx_cols(const SSIDXData &ssidx_data, AsciiTable &at, int r, int c) { // // Skill Score Index // Dump out the SSIDX line: - // FCST_MODEL, REF_MODEL, N_TERM, - // N_VLD, SS_INDEX + // FCST_MODEL, REF_MODEL, N_INIT, + // N_TERM, N_VLD, SS_INDEX // at.set_entry(r, c+0, // Forecast model name - ssidx_info.fcst_model); + ssidx_data.fcst_model); at.set_entry(r, c+1, // Reference model name - ssidx_info.ref_model); + ssidx_data.ref_model); - at.set_entry(r, c+2, // Number of terms - ssidx_info.n_term); + at.set_entry(r, c+2, // Number of initializations + ssidx_data.init_time.n()); - at.set_entry(r, c+3, // Number of valid terms - ssidx_info.n_vld); + at.set_entry(r, c+3, // Number of terms + ssidx_data.n_term); - at.set_entry(r, c+4, // Skill score index value - ssidx_info.ss_index); + at.set_entry(r, c+4, // Number of valid terms + ssidx_data.n_vld); + + at.set_entry(r, c+5, // Skill score index value + ssidx_data.ss_index); return; } diff --git a/met/src/libcode/vx_stat_out/stat_columns.h b/met/src/libcode/vx_stat_out/stat_columns.h index bde05f3ffb..96a3a0c5c4 100644 --- a/met/src/libcode/vx_stat_out/stat_columns.h +++ b/met/src/libcode/vx_stat_out/stat_columns.h @@ -189,7 +189,7 @@ extern void write_ssvar_cols (const PairDataEnsemble *, int, double, AsciiTable &, int, int); extern void write_relp_cols (const PairDataEnsemble *, AsciiTable &, int, int); -extern void write_ssidx_cols (const SSIndexInfo, +extern void write_ssidx_cols (const SSIDXData &, AsciiTable &, int, int); // Setup column justification for STAT AsciiTable objects diff --git a/met/src/libcode/vx_statistics/met_stats.h b/met/src/libcode/vx_statistics/met_stats.h index 4a2a5ac780..69824fa898 100644 --- a/met/src/libcode/vx_statistics/met_stats.h +++ b/met/src/libcode/vx_statistics/met_stats.h @@ -666,14 +666,26 @@ inline double DMAPInfo::get_beta_value() const { return(beta_value); } //////////////////////////////////////////////////////////////////////// // -// Structure to store the Skill Score Index +// Structure to store the Skill Score Index output // //////////////////////////////////////////////////////////////////////// -struct SSIndexInfo { +struct SSIDXData { + + // Index name ConcatString name; - ConcatString fcst_model, ref_model; + + // Forecast and reference model names + ConcatString fcst_model; + ConcatString ref_model; + + // List of unique initialization times + TimeArray init_time; + + // Number of terms and number valid int n_term, n_vld; + + // Skill score index value double ss_index; }; diff --git a/test/hdr/met_10_1.hdr b/test/hdr/met_10_1.hdr index 1c1866430e..641ac25813 100644 --- a/test/hdr/met_10_1.hdr +++ b/test/hdr/met_10_1.hdr @@ -29,7 +29,7 @@ VL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L VAL1L2 : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL UFABAR VFABAR UOABAR VOABAR UVFOABAR UVFFABAR UVOOABAR VCNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FBAR FBAR_BCL FBAR_BCU OBAR OBAR_BCL OBAR_BCU FS_RMS FS_RMS_BCL FS_RMS_BCU OS_RMS OS_RMS_BCL OS_RMS_BCU MSVE MSVE_BCL MSVE_BCU RMSVE RMSVE_BCL RMSVE_BCU FSTDEV FSTDEV_BCL FSTDEV_BCU OSTDEV OSTDEV_BCL OSTDEV_BCU FDIR FDIR_BCL FDIR_BCU ODIR ODIR_BCL ODIR_BCU FBAR_SPEED FBAR_SPEED_BCL FBAR_SPEED_BCU OBAR_SPEED OBAR_SPEED_BCL OBAR_SPEED_BCU VDIFF_SPEED VDIFF_SPEED_BCL VDIFF_SPEED_BCU VDIFF_DIR VDIFF_DIR_BCL VDIFF_DIR_BCU SPEED_ERR SPEED_ERR_BCL SPEED_ERR_BCU SPEED_ABSERR SPEED_ABSERR_BCL SPEED_ABSERR_BCU DIR_ERR DIR_ERR_BCL DIR_ERR_BCU DIR_ABSERR DIR_ABSERR_BCL DIR_ABSERR_BCU GENMPR : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX STORM_ID AGEN_INIT AGEN_FHR AGEN_LAT AGEN_LON AGEN_DLAND BGEN_LAT BGEN_LON BGEN_DLAND GEN_DIST GEN_TDIFF INIT_TDIFF DEV_CAT OPS_CAT -SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE FCST_MODEL REF_MODEL N_TERM N_VLD SS_INDEX +SSIDX : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE FCST_MODEL REF_MODEL N_INIT N_TERM N_VLD SS_INDEX MODE_SOA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES OBJECT_ID OBJECT_CAT CENTROID_X CENTROID_Y CENTROID_LAT CENTROID_LON AXIS_ANG LENGTH WIDTH AREA AREA_THRESH CURVATURE CURVATURE_X CURVATURE_Y COMPLEXITY INTENSITY_10 INTENSITY_25 INTENSITY_50 INTENSITY_75 INTENSITY_90 INTENSITY_50 INTENSITY_SUM MODE_POA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES OBJECT_ID OBJECT_CAT CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST MODE_CTS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS From 271d5288632bc59ae684832db82478f39399adf3 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 12:22:46 -0600 Subject: [PATCH 22/60] Per #1788, update STATAnalysisJob to explicitly set the job type directly rather than parsing a string. --- met/src/libcode/vx_analysis_util/stat_job.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/met/src/libcode/vx_analysis_util/stat_job.h b/met/src/libcode/vx_analysis_util/stat_job.h index b075ffc9ed..117fd58d6f 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.h +++ b/met/src/libcode/vx_analysis_util/stat_job.h @@ -131,6 +131,7 @@ class STATAnalysisJob { void parse_job_command(const char *); void add_column_thresh(const char *, const char *); + void set_job_type (const STATJobType); int set_job_type (const char *); void set_dump_row (const char *); void set_stat_file(const char *); @@ -345,6 +346,8 @@ class STATAnalysisJob { //////////////////////////////////////////////////////////////////////// +inline void STATAnalysisJob::set_job_type (const STATJobType t) { job_type = t; return; } + inline void STATAnalysisJob::set_precision (int p) { precision = p; return; } inline int STATAnalysisJob::get_precision () const { return(precision); } From d3fb64477d38518d46a9ee9065454616a0dd0457 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 12:23:51 -0600 Subject: [PATCH 23/60] Per #1788, major change here adding skill_score_index_job.h/.cc to handle the computation of the skill score indices. --- met/src/tools/core/stat_analysis/Makefile.am | 6 +- .../stat_analysis/skill_score_index_job.cc | 329 ++++++++++++++++++ .../stat_analysis/skill_score_index_job.h | 108 ++++++ 3 files changed, 441 insertions(+), 2 deletions(-) create mode 100644 met/src/tools/core/stat_analysis/skill_score_index_job.cc create mode 100644 met/src/tools/core/stat_analysis/skill_score_index_job.h diff --git a/met/src/tools/core/stat_analysis/Makefile.am b/met/src/tools/core/stat_analysis/Makefile.am index 60ca332858..00e6a2488d 100644 --- a/met/src/tools/core/stat_analysis/Makefile.am +++ b/met/src/tools/core/stat_analysis/Makefile.am @@ -14,7 +14,8 @@ bin_PROGRAMS = stat_analysis stat_analysis_SOURCES = stat_analysis.cc \ aggr_stat_line.cc \ parse_stat_line.cc \ - stat_analysis_job.cc + stat_analysis_job.cc \ + skill_score_index_job.cc stat_analysis_CPPFLAGS = ${MET_CPPFLAGS} stat_analysis_LDFLAGS = ${MET_LDFLAGS} stat_analysis_LDADD = -lvx_stat_out \ @@ -47,4 +48,5 @@ stat_analysis_LDADD = -lvx_stat_out \ EXTRA_DIST = stat_analysis.h \ parse_stat_line.h \ aggr_stat_line.h \ - stat_analysis_job.h + stat_analysis_job.h \ + skill_score_index_job.h diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.cc b/met/src/tools/core/stat_analysis/skill_score_index_job.cc new file mode 100644 index 0000000000..80c49afc13 --- /dev/null +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.cc @@ -0,0 +1,329 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// +// +// Filename: skill_score_index_job.cc +// +// Description: +// +// Mod# Date Name Description +// ---- ---- ---- ----------- +// 000 07/28/21 Halley Gotway New +// +//////////////////////////////////////////////////////////////////////// + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "skill_score_index_job.h" +#include "parse_stat_line.h" + +//////////////////////////////////////////////////////////////////////// +// +// Code for class SSIndexJobInfo +// +//////////////////////////////////////////////////////////////////////// + +SSIndexJobInfo::SSIndexJobInfo() { + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +SSIndexJobInfo::~SSIndexJobInfo() { + clear(); +} + +//////////////////////////////////////////////////////////////////////// + +SSIndexJobInfo::SSIndexJobInfo(const SSIndexJobInfo &c) { + + init_from_scratch(); + + assign(c); +} + +//////////////////////////////////////////////////////////////////////// + +SSIndexJobInfo & SSIndexJobInfo::operator=(const SSIndexJobInfo &c) { + + if(this == &c) return(*this); + + assign(c); + + return(*this); +} + +//////////////////////////////////////////////////////////////////////// + +void SSIndexJobInfo::init_from_scratch() { + + clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void SSIndexJobInfo::clear() { + + // Initialize + name.clear(); + fcst_model.clear(); + ref_model.clear(); + n_term = 0; + n_fcst_lines.clear(); + n_ref_lines.clear(); + job_lt.clear(); + fcst_job.clear(); + ref_job.clear(); + fcst_sl1l2.clear(); + ref_sl1l2.clear(); + fcst_cts.clear(); + ref_cts.clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void SSIndexJobInfo::assign(const SSIndexJobInfo &c) { + + name = c.name; + fcst_model = c.fcst_model; + ref_model = c.ref_model; + n_term = c.n_term; + n_fcst_lines = c.n_fcst_lines; + n_ref_lines = c.n_ref_lines; + job_lt = c.job_lt; + fcst_job = c.fcst_job; + ref_job = c.ref_job; + fcst_sl1l2 = c.fcst_sl1l2; + ref_sl1l2 = c.ref_sl1l2; + fcst_cts = c.fcst_cts; + ref_cts = c.ref_cts; + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void SSIndexJobInfo::add_term(const STATAnalysisJob &fcst_term, + const STATAnalysisJob &ref_term) { + + // Increment the counter + n_term++; + + // Add the jobs + fcst_job.push_back(fcst_term); + ref_job.push_back(ref_term); + + // Add the input line type for each job + job_lt.push_back(string_to_statlinetype(fcst_term.line_type[0].c_str())); + + // Add partial sums + SL1L2Info sl1l2_info; + fcst_sl1l2.push_back(sl1l2_info); + ref_sl1l2.push_back(sl1l2_info); + + // Add contingency tables + CTSInfo cts_info; + fcst_cts.push_back(cts_info); + ref_cts.push_back(cts_info); + + // Add line counters + n_fcst_lines.add(0); + n_ref_lines.add(0); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +bool SSIndexJobInfo::is_keeper(const STATLine &line) { + bool keep = false; + + // Check each forecast and reference job + for(int i=0; i " + << "unxpected line type of " << statlinetype_to_string(job_lt[i]) + << "!\n\n"; + exit(1); + } + + // Check for conditions when a skill score cannot be computed + if(nint(n_fcst_lines[i]) == 0 || nint(n_ref_lines[i]) == 0 || + is_bad_data(fcst_stat) || is_bad_data(ref_stat) || + is_eq(ref_stat, 0.0)) { + ss = bad_data_double; + } + // Compute the skill score and keep a running sum of the skill + // scores and weights + else { + ss = 1.0 - (fcst_stat*fcst_stat)/(ref_stat*ref_stat); + ss_sum += ss*fcst_job[i].weight[0]; + weight_sum += fcst_job[i].weight[0]; + } + + // Print debug info for each term + mlog << Debug(3) << name << " Term " << i+1 + << ": " << fcst_job[i].fcst_var[0] << "/" + << fcst_job[i].fcst_lev[0] << " at " + << sec_to_hhmmss(fcst_job[i].fcst_lead[0]) << " lead, " + << n_fcst_lines[i] << " fcst " + << fcst_job[i].line_type[0] << "->" + << fcst_job[i].column[0] << " = " << fcst_stat << ", " + << n_ref_lines[i] << " ref " + << fcst_job[i].line_type[0] << "->" + << fcst_job[i].column[0] << " = " << ref_stat + << ", skill = " << ss + << ", weight = " << fcst_job[i].weight[0] << "\n"; + + // Check if the number of aggregated lines differ + if(nint(n_fcst_lines[i]) != nint(n_ref_lines[i])) { + mlog << Warning << "\nSSIndexJobInfo::compute_ss_index() -> " + << "the number of aggregated forecast and reference lines " + << "differ (" << n_fcst_lines[i] << " != " << n_ref_lines[i] + << ") for term " << i+1 << ".\n\n"; + } + + if(is_bad_data(ss)) { + mlog << Warning << "\nSSIndexJobInfo::compute_ss_index() -> " + << "can't compute the skill score for term " << i+1 << ".\n\n"; + } + else { + n_vld++; + } + + } // end for i + + // Compute the weighted average of the skill scores + if(is_eq(weight_sum, 0.0)) ss_avg = bad_data_double; + else ss_avg = ss_sum/weight_sum; + + // Store the output data + data.name = name; + data.fcst_model = fcst_model; + data.ref_model = ref_model; + data.init_time = init_time; + data.n_term = n_term; + data.n_vld = n_vld; + if(is_bad_data(ss_avg) || is_eq(ss_avg, 1.0)) { + data.ss_index = bad_data_double; + } + else { + data.ss_index = sqrt(1.0/(1.0 - ss_avg)); + } + + mlog << Debug(3) + << name << " Weighted Average = " << ss_avg << "\n" + << name << " Skill Score Value = " << data.ss_index << "\n"; + + return(data); +} + +//////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.h b/met/src/tools/core/stat_analysis/skill_score_index_job.h new file mode 100644 index 0000000000..c1d52bfaf3 --- /dev/null +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.h @@ -0,0 +1,108 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2021 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// +// +// Filename: skill_score_index_job.h +// +// Description: +// +// Mod# Date Name Description +// ---- ---- ---- ----------- +// 000 07/28/21 Halley Gotway New +// +//////////////////////////////////////////////////////////////////////// + +#ifndef __SKILL_SCORE_INDEX_JOB_H__ +#define __SKILL_SCORE_INDEX_JOB_H__ + +//////////////////////////////////////////////////////////////////////// + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "met_stats.h" +#include "stat_job.h" +#include "vx_config.h" +#include "vx_util.h" + +//////////////////////////////////////////////////////////////////////// +// +// Class used to compute the Skill Score Index Job +// +//////////////////////////////////////////////////////////////////////// + +class SSIndexJobInfo { + + private: + void init_from_scratch(); + void assign(const SSIndexJobInfo &); + + public: + + SSIndexJobInfo(); + ~SSIndexJobInfo(); + SSIndexJobInfo(const SSIndexJobInfo &); + SSIndexJobInfo & operator=(const SSIndexJobInfo &); + + // Index name + ConcatString name; + + // Forecast and reference model names + ConcatString fcst_model, ref_model; + + // Array of unique initialization times + TimeArray init_time; + + // Number of terms + int n_term; + + // Array of input lines counts for each term + NumArray n_fcst_lines, n_ref_lines; + + // Input line type for each term + vector job_lt; + + // Vectors of jobs for each term + vector fcst_job; + vector ref_job; + + // Vectors of partial sums and contingency tables for each term + vector fcst_sl1l2; + vector ref_sl1l2; + vector fcst_cts; + vector ref_cts; + + // Add jobs for each term + void add_term(const STATAnalysisJob &, + const STATAnalysisJob &); + + // Process STAT lines + bool is_keeper(const STATLine &); + bool add(STATLine &); + + // Compute the skill score index value + SSIDXData compute_ss_index(); + + void clear(); +}; + +//////////////////////////////////////////////////////////////////////// + +#endif // __SKILL_SCORE_INDEX_JOB__ + +//////////////////////////////////////////////////////////////////////// From c2a0796ff372dac4df36779af0dd772bd8b9098f Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 12:25:26 -0600 Subject: [PATCH 24/60] Per #1788, update stat_analysis source code to make use of the new classes in skill_score_index_job.h/.cc. --- .../core/stat_analysis/aggr_stat_line.cc | 182 ++++- .../tools/core/stat_analysis/aggr_stat_line.h | 9 +- .../core/stat_analysis/stat_analysis_job.cc | 621 ++---------------- 3 files changed, 257 insertions(+), 555 deletions(-) diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.cc b/met/src/tools/core/stat_analysis/aggr_stat_line.cc index 3d2635969e..7b39bd3b12 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -1558,7 +1558,6 @@ void aggr_psum_lines(LineDataFile &f, STATAnalysisJob &job, // Increment sums in the existing map entry // else { - m[key].sl1l2_info += cur_sl1l2; m[key].vl1l2_info += cur_vl1l2; m[key].nbrcnt_info += cur_nbrcnt; @@ -3397,6 +3396,187 @@ void aggr_time_series_lines(LineDataFile &f, STATAnalysisJob &job, //////////////////////////////////////////////////////////////////////// +void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, + const ConcatString &name, + map &m, + int &n_in, int &n_out) { + STATLine line; + AggrSSIndexInfo cur; + ConcatString key; + int i, n_term; + + // + // Store the index name + // + cur.job_info.name = name; + + // + // Check that the -model option has been supplied exactly 2 times. + // The first is the forecast model and the second is the reference. + // + if(job.model.n() != 2) { + mlog << Error << "\naggr_ss_index() -> " + << "this job may only be called when the \"-model\" option " + << "has been used exactly twice to specify the forecast " + << "model followed by the reference model.\n\n"; + throw(1); + } + else { + cur.job_info.fcst_model = job.model[0]; + cur.job_info.ref_model = job.model[1]; + } + + mlog << Debug(3) + << "Computing " << cur.job_info.name << " for forecast model (" + << cur.job_info.fcst_model << ") versus reference model (" + << cur.job_info.ref_model << ").\n"; + + // + // Use the length of the fcst_var array to infer the number of terms. + // + if((n_term = job.fcst_var.n()) < 1) { + mlog << Error << "\naggr_ss_var() -> " + << "you must define the Skill Score Index to be computed " + << "using the \"-fcst_var\", \"-fcst_lev\", \"-fcst_lead\", " + << "\"-line_type\", \"-column\", and \"-weight\" options.\n\n"; + throw(1); + } + + // + // Check that the required elements are of the same length. + // + if(n_term != job.fcst_lev.n() || + n_term != job.fcst_lead.n() || + n_term != job.line_type.n() || + n_term != job.column.n() || + n_term != job.weight.n()) { + mlog << Error << "\ncompute_ss_index() -> " + << "all filtering parameters for defining the Skill Score " + << "Index must be of the same length. Check \"-fcst_var\", " + << "\"-fcst_lev\", \"-fcst_lead\", \"-line_type\", " + << "\"-column\", and \"-weight\" options.\n\n"; + throw(1); + } + + // + // Create a job for each term + // + for(i=0; i " + << "a Skill Score Index can only be computed using " + << "statistics derived from SL1L2 or CTC line types." + << "\n\n"; + throw(1); + } + } + + // + // Set filtering options + // + fcst_term.model.set(cur.job_info.fcst_model); + if(job.fcst_lead.n() == n_term) fcst_term.fcst_lead.set(job.fcst_lead[i]); + if(job.obs_lead.n() == n_term) fcst_term.obs_lead.set(job.obs_lead[i]); + if(job.fcst_init_hour.n() == n_term) fcst_term.fcst_init_hour.set(job.fcst_init_hour[i]); + if(job.obs_init_hour.n() == n_term) fcst_term.obs_init_hour.set(job.obs_init_hour[i]); + if(job.fcst_var.n() == n_term) fcst_term.fcst_var.set(job.fcst_var[i]); + if(job.obs_var.n() == n_term) fcst_term.obs_var.set(job.obs_var[i]); + if(job.fcst_lev.n() == n_term) fcst_term.fcst_lev.add(job.fcst_lev[i]); + if(job.obs_lev.n() == n_term) fcst_term.obs_lev.set(job.obs_lev[i]); + if(job.obtype.n() == n_term) fcst_term.obtype.set(job.obtype[i]); + if(job.vx_mask.n() == n_term) fcst_term.vx_mask.set(job.vx_mask[i]); + if(job.interp_mthd.n() == n_term) fcst_term.interp_mthd.set(job.interp_mthd[i]); + if(job.interp_pnts.n() == n_term) fcst_term.interp_pnts.set(job.interp_pnts[i]); + if(job.fcst_thresh.n() == n_term) fcst_term.fcst_thresh.set(job.fcst_thresh[i]); + if(job.obs_thresh.n() == n_term) fcst_term.obs_thresh.set(job.obs_thresh[i]); + if(job.column.n() == n_term) fcst_term.column.set(job.column[i]); + if(job.weight.n() == n_term) fcst_term.weight.set(job.weight[i]); + + // + // Set the reference model job identical to the forecast model + // job but with a different model name. + // + ref_term = fcst_term; + ref_term.model.set(cur.job_info.ref_model); + + // + // Store the forecast and reference jobs for this term + // + cur.job_info.add_term(fcst_term, ref_term); + + } // end for i + + // + // Process the STAT lines + // + while(f >> line) { + + if(line.is_header()) continue; + + n_in++; + + if(cur.job_info.is_keeper(line)) { + + // + // Build the map key for the current line + // + key = job.get_case_info(line); + + // + // Add a new map entry, if necessary + // + if(m.count(key) == 0) { + m[key] = cur; + mlog << Debug(3) << "[Case " << m.size() + << "] Added new case for key \"" + << key << "\".\n"; + } + + // + // Add the current line to the map entry + // + m[key].job_info.add(line); + + // + // Keep track of the unique header column entries + // + m[key].hdr.add(line); + + // + // Write line to dump file + // + job.dump_stat_line(line); + + n_out++; + } + } // end while + + return; +} + +//////////////////////////////////////////////////////////////////////// + void mpr_to_ctc(STATAnalysisJob &job, const AggrMPRInfo &info, int i_thresh, CTSInfo &cts_info) { int i; diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.h b/met/src/tools/core/stat_analysis/aggr_stat_line.h index 4e8d5e0461..d07f9a7d87 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.h +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.h @@ -53,6 +53,7 @@ using namespace std; #include "vx_statistics.h" #include "vx_stat_out.h" #include "ens_stats.h" +#include "skill_score_index_job.h" //////////////////////////////////////////////////////////////////////// @@ -162,7 +163,7 @@ struct AggrRPSInfo { struct AggrSSIndexInfo { StatHdrInfo hdr; - SSIndexInfo info; + SSIndexJobInfo job_info; }; // Define struct used to perform comparisons for SSVAR bins @@ -292,6 +293,12 @@ extern void aggr_time_series_lines( map &, int &, int &); +extern void aggr_ss_index( + LineDataFile &, STATAnalysisJob &, + const ConcatString &, + map &, + int &, int &); + //////////////////////////////////////////////////////////////////////// extern void mpr_to_ctc( diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index c3a83a635e..72e6527247 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -190,7 +190,10 @@ void do_job(const ConcatString &jobstring, STATAnalysisJob &job, job.job_type != stat_job_summary && job.job_type != stat_job_aggr && job.job_type != stat_job_aggr_stat && - job.job_type != stat_job_ramp) { + job.job_type != stat_job_ramp && + job.job_type != stat_job_go_index && + job.job_type != stat_job_cbs_index && + job.job_type != stat_job_ss_index) { mlog << Warning << "\nThe -by option is ignored for the \"" << statjobtype_to_string(job.job_type) << "\" job type.\n\n"; } @@ -367,7 +370,7 @@ void do_job_summary(const ConcatString &jobstring, LineDataFile &f, // Check for no matching STAT lines // if(n_out == 0) { - mlog << Warning << "\ndo_job_aggr() -> " + mlog << Warning << "\ndo_job_summary() -> " << "no matching STAT lines found for job: " << jobstring << "\n\n"; return; @@ -3933,16 +3936,18 @@ void write_job_ramp_cols(const STATAnalysisJob &job, AsciiTable &at, //////////////////////////////////////////////////////////////////////// void write_job_ss_index(STATAnalysisJob &job, - AggrSSIndexInfo &ssidx_info, + map &m, AsciiTable &at) { + map::iterator it; int n_row, n_col, r, c; StatHdrColumns shc; + SSIDXData ssidx_data; // // Setup the output table // - n_row = 2; - n_col = 1 + n_ssidx_columns; + n_row = 1 + m.size(); + n_col = 1 + job.by_column.n() + n_ssidx_columns; // // Prepare the output @@ -3952,49 +3957,59 @@ void write_job_ss_index(STATAnalysisJob &job, } else { write_job_aggr_hdr(job, n_row, n_col, at); - write_header_row(ssidx_columns, n_ssidx_columns, 0, at, 0, 1); + c = 1 + job.by_column.n(); + write_header_row(ssidx_columns, n_ssidx_columns, 0, at, 0, c); } - mlog << Debug(2) << "Computing output for 1 case(s).\n"; + mlog << Debug(2) << "Computing output for " + << (int) m.size() << " case(s).\n"; // - // Write the output STAT header columns + // Loop through the map // - ConcatString cur_case(""); + for(it = m.begin(), r=1; it != m.end(); it++) { - shc = ssidx_info.hdr.get_shc(cur_case, job.by_column, - job.hdr_name, job.hdr_value, stat_ssidx); + // + // Format the output STAT header columns + // + shc = it->second.hdr.get_shc(it->first, job.by_column, + job.hdr_name, job.hdr_value, stat_ssidx); - // - // Set FCST/OBS_VAR = skill score index type - // Set FCST/OBS_UNITS = FCST/OBS_LEV = NA - // - shc.set_fcst_var(ssidx_info.info.name); - shc.set_obs_var(ssidx_info.info.name); - shc.set_fcst_units(na_str); - shc.set_obs_units(na_str); - shc.set_fcst_lev(na_str); - shc.set_obs_lev(na_str); + // + // Set FCST/OBS_VAR = skill score index type + // Set FCST/OBS_UNITS = FCST/OBS_LEV = NA + // + shc.set_fcst_var(it->second.job_info.name); + shc.set_obs_var(it->second.job_info.name); + shc.set_fcst_units(na_str); + shc.set_obs_units(na_str); + shc.set_fcst_lev(na_str); + shc.set_obs_lev(na_str); - // - // Initialize - // - r = 1; - c = 0; + // + // Retrieve the SSIDXData + // + ssidx_data = it->second.job_info.compute_ss_index(); - // - // SSIDX output line - // - if(job.stat_out) { - write_header_cols(shc, job.stat_at, job.stat_row); - write_ssidx_cols(ssidx_info.info, job.stat_at, - job.stat_row++, n_header_columns); - } - else { - at.set_entry(r, c++, ssidx_info.info.name); - write_case_cols(cur_case, at, r, c); - write_ssidx_cols(ssidx_info.info, at, r++, c); - } + // + // Initialize + // + c = 0; + + // + // SSIDX output line + // + if(job.stat_out) { + write_header_cols(shc, job.stat_at, job.stat_row); + write_ssidx_cols(ssidx_data, job.stat_at, + job.stat_row++, n_header_columns); + } + else { + at.set_entry(r, c++, ssidx_data.name); + write_case_cols(it->first, at, r, c); + write_ssidx_cols(ssidx_data, at, r++, c); + } + } // end for it return; } @@ -4020,7 +4035,8 @@ void write_job_ss_index(STATAnalysisJob &job, void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, STATAnalysisJob &job, int &n_in, int &n_out, ofstream *sa_out) { - AggrSSIndexInfo ssidx_info; + map ssidx_map; + ConcatString name; AsciiTable out_at; // @@ -4031,14 +4047,19 @@ void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, // // Determine the job type // - if(job.job_type == stat_job_go_index) ssidx_info.info.name = "GO_INDEX"; - else if(job.job_type == stat_job_cbs_index) ssidx_info.info.name = "CBS_INDEX"; - else ssidx_info.info.name = "SS_INDEX"; + if(job.job_type == stat_job_go_index) name = "GO_INDEX"; + else if(job.job_type == stat_job_cbs_index) name = "CBS_INDEX"; + else name = "SS_INDEX"; // // Compute the Skill Score Index // - compute_ss_index(f, job, ssidx_info, n_in, n_out); + aggr_ss_index(f, job, name, ssidx_map, n_in, n_out); + + // + // Write the output + // + write_job_ss_index(job, ssidx_map, out_at); // // Check for no matching STAT lines @@ -4051,15 +4072,11 @@ void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, } // - // Write the output - // - write_job_ss_index(job, ssidx_info, out_at); - - // - // Write the Ascii Table and the job command line + // Write the ASCII Table and the job command line + // If -out_stat was specified, do not write output // write_jobstring(jobstring, sa_out); - write_table(out_at, sa_out); + if(!job.stat_out) write_table(out_at, sa_out); return; } @@ -4252,508 +4269,6 @@ void write_line(const ConcatString &str, ofstream *sa_out) { //////////////////////////////////////////////////////////////////////// -void compute_ss_index(LineDataFile &f, STATAnalysisJob &job, - AggrSSIndexInfo &ssidx_info, - int &n_in, int &n_out) { - STATLine line; - SL1L2Info sl1l2; - TTContingencyTable ct; - CNTInfo fcst_cnt, ref_cnt; - bool keep; - int i, n_term, n_vld; - double fcst_stat, ref_stat, ss, ss_sum, weight_sum; - double ss_avg; - - // - // Check that the -model option has been supplied exactly 2 times. - // The first is the forecast model and the second is the reference. - // - if(job.model.n() != 2) { - mlog << Error << "\ncompute_ss_index() -> " - << "this job may only be called when the \"-model\" option " - << "has been used exactly twice to specify the forecast " - << "model followed by the reference model.\n\n"; - throw(1); - } - else { - ssidx_info.info.fcst_model = job.model[0]; - ssidx_info.info.ref_model = job.model[1]; - } - - // - // Use the length of the fcst_var array to infer the number of terms. - // - if((n_term = job.fcst_var.n()) < 1) { - mlog << Error << "\ncompute_ss_index() -> " - << "you must define the Skill Score Index to be computed " - << "using the \"-fcst_var\", \"-fcst_lev\", \"-fcst_lead\", " - << "\"-line_type\", \"-column\", and \"-weight\" options.\n\n"; - throw(1); - } - - // - // Check that the required elements are of the same length. - // - if(n_term != job.fcst_lev.n() || - n_term != job.fcst_lead.n() || - n_term != job.line_type.n() || - n_term != job.column.n() || - n_term != job.weight.n()) { - mlog << Error << "\ncompute_ss_index() -> " - << "all filtering parameters for defining the Skill Score " - << "Index must be of the same length. Check \"-fcst_var\", " - << "\"-fcst_lev\", \"-fcst_lead\", \"-line_type\", " - << "\"-column\", and \"-weight\" options.\n\n"; - throw(1); - } - - // - // Define arrays of jobs for each term in the Skill Score Index. - // Separate arrays for the forecast and reference models. - // - STATAnalysisJob *fcst_job = (STATAnalysisJob *) 0, *ref_job = (STATAnalysisJob *) 0; - fcst_job = new STATAnalysisJob [n_term]; - ref_job = new STATAnalysisJob [n_term]; - - // - // Define arrays of objects to store the partial sums or contingency - // table counts for each term in the Skill Score Index. - // - SL1L2Info *fcst_sl1l2 = (SL1L2Info *) 0, *ref_sl1l2 = (SL1L2Info *) 0; - CTSInfo *fcst_cts = (CTSInfo *) 0, *ref_cts = (CTSInfo *) 0; - fcst_sl1l2 = new SL1L2Info [n_term]; - ref_sl1l2 = new SL1L2Info [n_term]; - fcst_cts = new CTSInfo [n_term]; - ref_cts = new CTSInfo [n_term]; - - // - // Define array of line types to be aggregated for each term in the - // Skill Score Index. - // - STATLineType *job_lt = (STATLineType *) 0; - job_lt = new STATLineType [n_term]; - - // - // Arrays to keep track of the number of stat lines per term - // - NumArray n_fcst_lines, n_ref_lines; - - mlog << Debug(3) - << "Forecast Model = " << ssidx_info.info.fcst_model << "\n" - << "Reference Model = " << ssidx_info.info.ref_model << "\n"; - - // - // Set up the job for each term in the index. - // - for(i=0; i " - << "a Skill Score Index can only be computed using " - << "statistics derived from SL1L2 or CTC line types." - << "\n\n"; - throw(1); - } - } - - // - // Set the reference model job identical to the forecast model - // job but with a different model name. - // - ref_job[i] = fcst_job[i]; - ref_job[i].model.set(ssidx_info.info.ref_model); - - } // end for i - - // - // Process the STAT lines - // - n_in = n_out = 0; - while(f >> line) { - - if(line.is_header()) continue; - - n_in++; - - // - // Loop through the jobs to see if this line should be kept - // - keep = 0; - for(i=0; i " - << "the number of aggregated forecast and reference lines " - << "differ (" << n_fcst_lines[i] << " != " << n_ref_lines[i] - << ") for term " << i+1 << ".\n\n"; - } - - if(is_bad_data(ss)) { - mlog << Warning << "\ncompute_ss_index() -> " - << "can't compute skill score for term " << i+1 << ".\n\n"; - } - else { - n_vld++; - } - - } // end for i - - // - // Compute the weighted average of the skill scores. - // - if(is_eq(weight_sum, 0.0)) ss_avg = bad_data_double; - else ss_avg = ss_sum/weight_sum; - - // - // Compute the Skill Score Index value. - // - ssidx_info.info.n_term = n_term; - ssidx_info.info.n_vld = n_vld; - if(is_bad_data(ss_avg) || is_eq(ss_avg, 1.0)) { - ssidx_info.info.ss_index = bad_data_double; - } - else { - ssidx_info.info.ss_index = sqrt(1.0/(1.0 - ss_avg)); - } - - mlog << Debug(3) << "Skill Score Index Weighted Average = " << ss_avg << "\n" - << "Skill Score Index Value = " << ssidx_info.info.ss_index << "\n"; - - // - // Clean up allocated memory. - // - if(fcst_job) { delete [] fcst_job; fcst_job = (STATAnalysisJob *) 0; } - if(ref_job) { delete [] ref_job; ref_job = (STATAnalysisJob *) 0; } - if(fcst_sl1l2) { delete [] fcst_sl1l2; fcst_sl1l2 = (SL1L2Info *) 0; } - if(ref_sl1l2) { delete [] ref_sl1l2; ref_sl1l2 = (SL1L2Info *) 0; } - if(fcst_cts) { delete [] fcst_cts; fcst_cts = (CTSInfo *) 0; } - if(ref_cts) { delete [] ref_cts; ref_cts = (CTSInfo *) 0; } - if(job_lt) { delete [] job_lt; job_lt = (STATLineType *) 0; } - - return; -} - -//////////////////////////////////////////////////////////////////////// - void write_case_cols(const ConcatString &cs, AsciiTable &at, int &r, int &c) { StringArray sa = cs.split(":"); From 3ac9c67fb19fb00ead2bc52f611a85bc715d7398 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 12:38:29 -0600 Subject: [PATCH 25/60] Per #1788, write a log message listing the model initialization times over which the skill score index is computed. --- met/src/basic/vx_cal/time_array.cc | 20 +++++++++++++++++++ met/src/basic/vx_cal/time_array.h | 6 ++++++ met/src/basic/vx_util/thresh_array.h | 1 - .../stat_analysis/skill_score_index_job.cc | 6 +++++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/met/src/basic/vx_cal/time_array.cc b/met/src/basic/vx_cal/time_array.cc index 4521693ed9..cbbf8aa8eb 100644 --- a/met/src/basic/vx_cal/time_array.cc +++ b/met/src/basic/vx_cal/time_array.cc @@ -567,4 +567,24 @@ return ( 0 ); } +//////////////////////////////////////////////////////////////////////// + + +ConcatString write_css(const TimeArray &ta) + +{ + +ConcatString css; + +for ( int i=0; i Date: Wed, 25 Aug 2021 12:46:40 -0600 Subject: [PATCH 26/60] Per #1788, tweak log message. --- met/src/tools/core/stat_analysis/skill_score_index_job.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.cc b/met/src/tools/core/stat_analysis/skill_score_index_job.cc index 0ce0ab0163..409cb5ca6a 100644 --- a/met/src/tools/core/stat_analysis/skill_score_index_job.cc +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.cc @@ -232,8 +232,8 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { SSIDXData data; mlog << Debug(3) - << "Computing " << name << " over " << init_time.n() - << " initialization times: " << write_css(init_time) << "\n"; + << "Computing " << name << " for " << init_time.n() + << " initialization time(s): " << write_css(init_time) << "\n"; // Compute a skill score for each term for(i=0, n_vld=0, ss_sum=weight_sum=0.0; i Date: Wed, 25 Aug 2021 17:18:36 -0600 Subject: [PATCH 27/60] Per #1788, compute n_term as the max lenght of the required arrays. --- .../core/stat_analysis/aggr_stat_line.cc | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.cc b/met/src/tools/core/stat_analysis/aggr_stat_line.cc index 7b39bd3b12..26994c9114 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -3432,29 +3432,39 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, << cur.job_info.ref_model << ").\n"; // - // Use the length of the fcst_var array to infer the number of terms. + // Compute the number of terms as the maximum array length // - if((n_term = job.fcst_var.n()) < 1) { + n_term = max( 0, job.fcst_var.n()); + n_term = max(n_term, job.fcst_lev.n()); + n_term = max(n_term, job.fcst_lead.n()); + n_term = max(n_term, job.line_type.n()); + n_term = max(n_term, job.column.n()); + n_term = max(n_term, job.weight.n()); + + // + // Must be at least one term + // + if(n_term < 1) { mlog << Error << "\naggr_ss_var() -> " - << "you must define the Skill Score Index to be computed " + << "you must define the skill score index to be computed " << "using the \"-fcst_var\", \"-fcst_lev\", \"-fcst_lead\", " << "\"-line_type\", \"-column\", and \"-weight\" options.\n\n"; throw(1); } // - // Check that the required elements are of the same length. + // Sanity check the array lengths // - if(n_term != job.fcst_lev.n() || - n_term != job.fcst_lead.n() || - n_term != job.line_type.n() || - n_term != job.column.n() || - n_term != job.weight.n()) { + if((job.fcst_lev.n() != n_term && job.fcst_lev.n() != 1) || + (job.fcst_lead.n() != n_term && job.fcst_lead.n() != 1) || + (job.line_type.n() != n_term && job.line_type.n() != 1) || + (job.column.n() != n_term && job.column.n() != 1) || + (job.weight.n() != n_term && job.weight.n() != 1)) { mlog << Error << "\ncompute_ss_index() -> " - << "all filtering parameters for defining the Skill Score " - << "Index must be of the same length. Check \"-fcst_var\", " - << "\"-fcst_lev\", \"-fcst_lead\", \"-line_type\", " - << "\"-column\", and \"-weight\" options.\n\n"; + << "each skill score index parameter must have the same length (" + << n_term << ") or have length 1!\n" + << "Check the \"-fcst_var\", \"-fcst_lev\", \"-fcst_lead\", " + << "\"-line_type\", \"-column\", and \"-weight\" options.\n\n"; throw(1); } @@ -3485,7 +3495,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, STATLineType lt = string_to_statlinetype(job.line_type[i].c_str()); if(lt != stat_sl1l2 && lt != stat_ctc) { mlog << Error << "\naggr_ss_index() -> " - << "a Skill Score Index can only be computed using " + << "a skill score index can only be computed using " << "statistics derived from SL1L2 or CTC line types." << "\n\n"; throw(1); From 8bc7b453bfa476dd4df2e56088daa3f8cfee6265 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 19:07:21 -0600 Subject: [PATCH 28/60] Per #1788, drop in the ACTUAL definition of the GO index provide by the DTC AF project team. Also simplify the existing arrays down to lenght one if the value remains constant. --- met/data/config/STATAnalysisConfig_CBS_Index | 118 +++++++------------ met/data/config/STATAnalysisConfig_GO_Index | 52 +++----- 2 files changed, 57 insertions(+), 113 deletions(-) diff --git a/met/data/config/STATAnalysisConfig_CBS_Index b/met/data/config/STATAnalysisConfig_CBS_Index index 62e2957674..a848235754 100644 --- a/met/data/config/STATAnalysisConfig_CBS_Index +++ b/met/data/config/STATAnalysisConfig_CBS_Index @@ -11,18 +11,14 @@ // model = []; -fcst_lead = [ "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48" +fcst_lead = [ "24", "48", "72", "96", "120", + "24", "48", "72", "96", "120", + "24", "48", "72", "96", "120", + "24", "48", "72", "96", "120", + "24", "48", "72", "96", "120", + "24", "48", "72", "96", "120", + "24", "48", "72", "96", "120", + "24", "48", "72", "96", "120" ]; obs_lead = []; @@ -50,39 +46,39 @@ obs_init_inc = []; obs_init_exc = []; obs_init_hour = []; -fcst_var = [ "WIND", "WIND", "WIND", "WIND", - "WIND", "WIND", "WIND", "WIND", - "WIND", "WIND", "WIND", "WIND", - "WIND", "WIND", "WIND", "WIND", - "DPT", "DPT", "DPT", "DPT", - "DPT", "DPT", "DPT", "DPT", - "DPT", "DPT", "DPT", "DPT", - "DPT", "DPT", "DPT", "DPT", - "TMP", "TMP", "TMP", "TMP", - "TMP", "TMP", "TMP", "TMP", - "HGT", "HGT", "HGT", "HGT", - "PRMSL", "PRMSL", "PRMSL", "PRMSL" +fcst_var = [ "PRMSL", "PRMSL", "PRMSL", "PRMSL", "PRMSL", + "HGT", "HGT", "HGT", "HGT", "HGT", + "WIND", "WIND", "WIND", "WIND", "WIND", + "WIND", "WIND", "WIND", "WIND", "WIND", + "WIND", "WIND", "WIND", "WIND", "WIND", + "PRMSL", "PRMSL", "PRMSL", "PRMSL", "PRMSL", + "HGT", "HGT", "HGT", "HGT", "HGT", + "WIND", "WIND", "WIND", "WIND", "WIND" ]; obs_var = []; -fcst_lev = [ "P250", "P250", "P250", "P250", - "P400", "P400", "P400", "P400", - "P850", "P850", "P850", "P850", - "Z10", "Z10", "Z10", "Z10", - "P400", "P400", "P400", "P400", - "P700", "P700", "P700", "P700", - "P850", "P850", "P850", "P850", - "Z2", "Z2", "Z2", "Z2", - "P400", "P400", "P400", "P400", - "Z2", "Z2", "Z2", "Z2", - "P400", "P400", "P400", "P400", - "Z0", "Z0", "Z0", "Z0" +fcst_lev = [ "Z0", "Z0", "Z0", "Z0", "Z0", + "P500", "P500", "P500", "P500", "P500", + "P250", "P250", "P250", "P250", "P250", + "P850", "P850", "P850", "P850", "P850", + "P250", "P250", "P250", "P250", "P250", + "Z0", "Z0", "Z0", "Z0", "Z0", + "P500", "P500", "P500", "P500", "P500", + "P250", "P250", "P250", "P250", "P250" ]; obs_lev = []; obtype = []; -vx_mask = []; +vx_mask = [ "NH", "NH", "NH", "NH", + "NH", "NH", "NH", "NH", + "NH", "NH", "NH", "NH", + "TROP", "TROP", "TROP", "TROP", + "TROP", "TROP", "TROP", "TROP", + "SH", "SH", "SH", "SH", + "SH", "SH", "SH", "SH", + "SH", "SH", "SH", "SH" + ]; interp_mthd = []; @@ -94,46 +90,18 @@ cov_thresh = []; alpha = []; -line_type = [ "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2" - ]; +line_type = [ "SL1L2" ]; -column = [ "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE" - ]; +column = [ "RMSE" ]; -weight = [ 4.0, 3.0, 2.0, 1.0, - 4.0, 3.0, 2.0, 1.0, - 4.0, 3.0, 2.0, 1.0, - 8.0, 6.0, 4.0, 2.0, - 8.0, 6.0, 4.0, 2.0, - 8.0, 6.0, 4.0, 2.0, - 8.0, 6.0, 4.0, 2.0, - 8.0, 6.0, 4.0, 2.0, - 4.0, 3.0, 2.0, 1.0, - 8.0, 6.0, 4.0, 2.0, - 4.0, 3.0, 2.0, 1.0, - 8.0, 6.0, 4.0, 2.0 +weight = [ 6.4, 6.4, 6.4, 6.4, + 2.4, 2.4, 2.4, 2.4, + 2.4, 2.4, 2.4, 2.4, + 2.0, 2.0, 2.0, 2.0, + 1.2, 1.2, 1.2, 1.2, + 3.2, 3.2, 3.2, 3.2, + 1.2, 1.2, 1.2, 1.2, + 1.2, 1.2, 1.2, 1.2 ]; //////////////////////////////////////////////////////////////////////////////// diff --git a/met/data/config/STATAnalysisConfig_GO_Index b/met/data/config/STATAnalysisConfig_GO_Index index 2b5ce181f5..0b4b5691e1 100644 --- a/met/data/config/STATAnalysisConfig_GO_Index +++ b/met/data/config/STATAnalysisConfig_GO_Index @@ -11,18 +11,18 @@ // model = []; -fcst_lead = [ "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48", - "12", "24", "36", "48" +fcst_lead = [ "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48" ]; obs_lead = []; @@ -94,33 +94,9 @@ cov_thresh = []; alpha = []; -line_type = [ "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2", - "SL1L2", "SL1L2", "SL1L2", "SL1L2" - ]; +line_type = [ "SL1L2" ]; -column = [ "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE", - "RMSE", "RMSE", "RMSE", "RMSE" - ]; +column = [ "RMSE" ]; weight = [ 4.0, 3.0, 2.0, 1.0, 4.0, 3.0, 2.0, 1.0, From f717cd5a9f97f1bd9006c0d0e52641ceaa3d1f68 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 19:19:42 -0600 Subject: [PATCH 29/60] Per #1788, correct CBS Index config file. Some parameters had 40 elements while others only had 32. --- met/data/config/STATAnalysisConfig_CBS_Index | 32 ++++++++++---------- met/data/config/STATAnalysisConfig_GO_Index | 24 +++++++-------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/met/data/config/STATAnalysisConfig_CBS_Index b/met/data/config/STATAnalysisConfig_CBS_Index index a848235754..45727261fd 100644 --- a/met/data/config/STATAnalysisConfig_CBS_Index +++ b/met/data/config/STATAnalysisConfig_CBS_Index @@ -70,14 +70,14 @@ obs_lev = []; obtype = []; -vx_mask = [ "NH", "NH", "NH", "NH", - "NH", "NH", "NH", "NH", - "NH", "NH", "NH", "NH", - "TROP", "TROP", "TROP", "TROP", - "TROP", "TROP", "TROP", "TROP", - "SH", "SH", "SH", "SH", - "SH", "SH", "SH", "SH", - "SH", "SH", "SH", "SH" +vx_mask = [ "NH", "NH", "NH", "NH", "NH", + "NH", "NH", "NH", "NH", "NH", + "NH", "NH", "NH", "NH", "NH", + "TROP", "TROP", "TROP", "TROP", "TROP", + "TROP", "TROP", "TROP", "TROP", "TROP", + "SH", "SH", "SH", "SH", "SH", + "SH", "SH", "SH", "SH", "SH", + "SH", "SH", "SH", "SH", "SH" ]; interp_mthd = []; @@ -94,14 +94,14 @@ line_type = [ "SL1L2" ]; column = [ "RMSE" ]; -weight = [ 6.4, 6.4, 6.4, 6.4, - 2.4, 2.4, 2.4, 2.4, - 2.4, 2.4, 2.4, 2.4, - 2.0, 2.0, 2.0, 2.0, - 1.2, 1.2, 1.2, 1.2, - 3.2, 3.2, 3.2, 3.2, - 1.2, 1.2, 1.2, 1.2, - 1.2, 1.2, 1.2, 1.2 +weight = [ 6.4, 6.4, 6.4, 6.4, 6.4, + 2.4, 2.4, 2.4, 2.4, 2.4, + 2.4, 2.4, 2.4, 2.4, 2.4, + 2.0, 2.0, 2.0, 2.0, 2.0, + 1.2, 1.2, 1.2, 1.2, 1.2, + 3.2, 3.2, 3.2, 3.2, 3.2, + 1.2, 1.2, 1.2, 1.2, 1.2, + 1.2, 1.2, 1.2, 1.2, 1.2 ]; //////////////////////////////////////////////////////////////////////////////// diff --git a/met/data/config/STATAnalysisConfig_GO_Index b/met/data/config/STATAnalysisConfig_GO_Index index 0b4b5691e1..09902f8194 100644 --- a/met/data/config/STATAnalysisConfig_GO_Index +++ b/met/data/config/STATAnalysisConfig_GO_Index @@ -98,18 +98,18 @@ line_type = [ "SL1L2" ]; column = [ "RMSE" ]; -weight = [ 4.0, 3.0, 2.0, 1.0, - 4.0, 3.0, 2.0, 1.0, - 4.0, 3.0, 2.0, 1.0, - 8.0, 6.0, 4.0, 2.0, - 8.0, 6.0, 4.0, 2.0, - 8.0, 6.0, 4.0, 2.0, - 8.0, 6.0, 4.0, 2.0, - 8.0, 6.0, 4.0, 2.0, - 4.0, 3.0, 2.0, 1.0, - 8.0, 6.0, 4.0, 2.0, - 4.0, 3.0, 2.0, 1.0, - 8.0, 6.0, 4.0, 2.0 +weight = [ 4.0, 3.0, 2.0, 1.0, + 4.0, 3.0, 2.0, 1.0, + 4.0, 3.0, 2.0, 1.0, + 8.0, 6.0, 4.0, 2.0, + 8.0, 6.0, 4.0, 2.0, + 8.0, 6.0, 4.0, 2.0, + 8.0, 6.0, 4.0, 2.0, + 8.0, 6.0, 4.0, 2.0, + 4.0, 3.0, 2.0, 1.0, + 8.0, 6.0, 4.0, 2.0, + 4.0, 3.0, 2.0, 1.0, + 8.0, 6.0, 4.0, 2.0 ]; //////////////////////////////////////////////////////////////////////////////// From 9353062929fe04a5f69a3fc6cf2f00ef6ae43ddc Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 19:25:07 -0600 Subject: [PATCH 30/60] Per #1788, do not write SSIDX output if there are no valid terms found. --- met/src/tools/core/stat_analysis/stat_analysis_job.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index 72e6527247..2580199d87 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -3991,6 +3991,11 @@ void write_job_ss_index(STATAnalysisJob &job, // ssidx_data = it->second.job_info.compute_ss_index(); + // + // Skip empty output + // + if(ssidx_data.n_vld == 0) continue; + // // Initialize // From 0668e77da3cfd828e26a4c05f77c7244030990ea Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 19:39:59 -0600 Subject: [PATCH 31/60] Per #1788, format the values in the skill score index log messages, replacing -9999 with NA. --- .../core/stat_analysis/skill_score_index_job.cc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.cc b/met/src/tools/core/stat_analysis/skill_score_index_job.cc index 409cb5ca6a..ca214f639c 100644 --- a/met/src/tools/core/stat_analysis/skill_score_index_job.cc +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.cc @@ -273,6 +273,15 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { weight_sum += fcst_job[i].weight[0]; } + // Format log message strings + ConcatString fcst_cs, ref_cs, ss_cs; + if(is_bad_data(fcst_stat)) fcst_cs << na_str; + else fcst_cs << fcst_stat; + if(is_bad_data(ref_stat)) ref_cs << na_str; + else ref_cs << ref_stat; + if(is_bad_data(ss)) ss_cs << na_str; + else ss_cs << ss; + // Print debug info for each term mlog << Debug(3) << name << " Term " << i+1 << ": " << fcst_job[i].fcst_var[0] << "/" @@ -280,11 +289,11 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { << sec_to_hhmmss(fcst_job[i].fcst_lead[0]) << " lead, " << n_fcst_lines[i] << " fcst " << fcst_job[i].line_type[0] << "->" - << fcst_job[i].column[0] << " = " << fcst_stat << ", " + << fcst_job[i].column[0] << " = " << fcst_cs << ", " << n_ref_lines[i] << " ref " << fcst_job[i].line_type[0] << "->" - << fcst_job[i].column[0] << " = " << ref_stat - << ", skill = " << ss + << fcst_job[i].column[0] << " = " << ref_cs + << ", skill = " << ss_cs << ", weight = " << fcst_job[i].weight[0] << "\n"; // Check if the number of aggregated lines differ From 3b14efb9c220904fc8c2e5ca12b6a6befb25d02b Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 19:41:12 -0600 Subject: [PATCH 32/60] Per #1788, update the get_stat() member functions. For 0 pairs, return bad data for all statistics other than TOTAL. --- met/src/libcode/vx_statistics/met_stats.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/met/src/libcode/vx_statistics/met_stats.cc b/met/src/libcode/vx_statistics/met_stats.cc index a1aa7870bf..17adde559a 100644 --- a/met/src/libcode/vx_statistics/met_stats.cc +++ b/met/src/libcode/vx_statistics/met_stats.cc @@ -431,6 +431,7 @@ void CTSInfo::compute_ci() { double CTSInfo::get_stat(const char *stat_name) { double v = bad_data_double; + // Find the statistic by name if(strcmp(stat_name, "TOTAL") == 0) v = cts.n(); else if(strcmp(stat_name, "BASER") == 0) v = cts.baser(); else if(strcmp(stat_name, "FMEAN") == 0) v = cts.fmean(); @@ -458,6 +459,11 @@ double CTSInfo::get_stat(const char *stat_name) { << "\"!\n\n"; exit(1); } + + // Return bad data for 0 pairs + if(cts.n() == 0 && strcmp(stat_name, "TOTAL") != 0) { + v = bad_data_double; + } return(v); } @@ -1001,6 +1007,7 @@ void CNTInfo::compute_ci() { double CNTInfo::get_stat(const char *stat_name) { double v = bad_data_double; + // Find the statistic by name if(strcmp(stat_name, "TOTAL" ) == 0) v = n; else if(strcmp(stat_name, "FBAR" ) == 0) v = fbar.v; else if(strcmp(stat_name, "FSTDEV" ) == 0) v = fstdev.v; @@ -1040,6 +1047,11 @@ double CNTInfo::get_stat(const char *stat_name) { exit(1); } + // Return bad data for 0 pairs + if(n == 0 && strcmp(stat_name, "TOTAL") != 0) { + v = bad_data_double; + } + return(v); } From d0ed0b49af0d4b9ce97197d2a44fc68fb646fa2e Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 19:49:52 -0600 Subject: [PATCH 33/60] Per #1788, working on readability of the warning and error messages. --- .../core/stat_analysis/skill_score_index_job.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.cc b/met/src/tools/core/stat_analysis/skill_score_index_job.cc index ca214f639c..548562e717 100644 --- a/met/src/tools/core/stat_analysis/skill_score_index_job.cc +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.cc @@ -274,7 +274,10 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { } // Format log message strings - ConcatString fcst_cs, ref_cs, ss_cs; + ConcatString term_cs, fcst_cs, ref_cs, ss_cs; + term_cs << fcst_job[i].fcst_var[0] << "/" + << fcst_job[i].fcst_lev[0] << " at " + << sec_to_hhmmss(fcst_job[i].fcst_lead[0]) << " lead"; if(is_bad_data(fcst_stat)) fcst_cs << na_str; else fcst_cs << fcst_stat; if(is_bad_data(ref_stat)) ref_cs << na_str; @@ -284,9 +287,7 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { // Print debug info for each term mlog << Debug(3) << name << " Term " << i+1 - << ": " << fcst_job[i].fcst_var[0] << "/" - << fcst_job[i].fcst_lev[0] << " at " - << sec_to_hhmmss(fcst_job[i].fcst_lead[0]) << " lead, " + << ": " << term_cs << ", " << n_fcst_lines[i] << " fcst " << fcst_job[i].line_type[0] << "->" << fcst_job[i].column[0] << " = " << fcst_cs << ", " @@ -301,12 +302,13 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { mlog << Warning << "\nSSIndexJobInfo::compute_ss_index() -> " << "the number of aggregated forecast and reference lines " << "differ (" << n_fcst_lines[i] << " != " << n_ref_lines[i] - << ") for term " << i+1 << ".\n\n"; + << ") for term " << i+1 << " (" << term_cs << ").\n\n"; } if(is_bad_data(ss)) { mlog << Warning << "\nSSIndexJobInfo::compute_ss_index() -> " - << "can't compute the skill score for term " << i+1 << ".\n\n"; + << "can't compute the skill score for term " << i+1 + << " (" << term_cs << ").\n\n"; } else { n_vld++; From 31f307015f4dde03286445300d53ce091c0cb700 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 21:44:40 -0600 Subject: [PATCH 34/60] Per #1788, define the ss_index_name directly in the STATAnalysis config file to make it really easy to modify. --- met/data/config/STATAnalysisConfig_CBS_Index | 1 + met/data/config/STATAnalysisConfig_GO_Index | 1 + met/data/config/STATAnalysisConfig_default | 1 + met/src/basic/vx_config/config_constants.h | 8 ++++++-- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/met/data/config/STATAnalysisConfig_CBS_Index b/met/data/config/STATAnalysisConfig_CBS_Index index 45727261fd..745c5efefa 100644 --- a/met/data/config/STATAnalysisConfig_CBS_Index +++ b/met/data/config/STATAnalysisConfig_CBS_Index @@ -121,6 +121,7 @@ boot = { //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "CBS_INDEX"; hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; diff --git a/met/data/config/STATAnalysisConfig_GO_Index b/met/data/config/STATAnalysisConfig_GO_Index index 09902f8194..8ce63fd944 100644 --- a/met/data/config/STATAnalysisConfig_GO_Index +++ b/met/data/config/STATAnalysisConfig_GO_Index @@ -129,6 +129,7 @@ boot = { //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "GO_INDEX"; hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; diff --git a/met/data/config/STATAnalysisConfig_default b/met/data/config/STATAnalysisConfig_default index 8ce7bde069..e1ec9912a0 100644 --- a/met/data/config/STATAnalysisConfig_default +++ b/met/data/config/STATAnalysisConfig_default @@ -108,6 +108,7 @@ wmo_fisher_stats = [ "CNT:PR_CORR", "CNT:SP_CORR", //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "SS_INDEX"; hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index fa9c30ec6f..3b78c305f8 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -791,6 +791,7 @@ static const char conf_key_vif_flag[] = "vif_flag"; static const char conf_key_wmo_sqrt_stats[] = "wmo_sqrt_stats"; static const char conf_key_wmo_fisher_stats[] = "wmo_fisher_stats"; static const char conf_key_jobs[] = "jobs"; +static const char conf_key_ss_index_name[] = "ss_index_name"; // // MODE specific parameter key names @@ -1068,7 +1069,10 @@ static const char conf_key_out_valid_mask[] = "out_valid_mask"; static const char conf_key_filter[] = "filter"; static const char conf_key_dland_thresh[] = "dland_thresh"; -// TC-Gen config options +// +// TC-Gen specific parameter key names +// + static const char conf_key_init_freq[] = "init_freq"; static const char conf_key_valid_freq[] = "valid_freq"; static const char conf_key_fcst_hr_window[] = "fcst_hr_window"; @@ -1102,6 +1106,7 @@ static const char conf_key_nc_pairs_grid[] = "nc_pairs_grid"; // // TC-RMW specific parameter key names // + static const char conf_key_n_range[] = "n_range"; static const char conf_key_n_azimuth[] = "n_azimuth"; static const char conf_key_max_range[] = "max_range_km"; @@ -1207,7 +1212,6 @@ static const int default_grib1_ptv = 2; static const int default_grib1_center = 7; static const int default_grib1_subcenter = 1; - //////////////////////////////////////////////////////////////////////// #endif // __CONFIG_CONSTANTS_H__ From 75a3a4997990ddb426947a3363b43efbbf24a720 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 21:46:05 -0600 Subject: [PATCH 35/60] Per #1788, store the ss_index_name configuration option in the StatJob class. Also support a command line -ss_index_name option to override the config file value. --- met/src/libcode/vx_analysis_util/stat_job.cc | 19 +++++++++++++++++++ met/src/libcode/vx_analysis_util/stat_job.h | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/met/src/libcode/vx_analysis_util/stat_job.cc b/met/src/libcode/vx_analysis_util/stat_job.cc index 0581b583d3..90b2d97500 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.cc +++ b/met/src/libcode/vx_analysis_util/stat_job.cc @@ -202,6 +202,8 @@ void STATAnalysisJob::clear() { boot_interval = bad_data_int; boot_rep_prop = bad_data_double; n_boot_rep = bad_data_int; + + ss_index_name.clear(); hss_ec_value = bad_data_double; rank_corr_flag = false; vif_flag = false; @@ -345,6 +347,7 @@ void STATAnalysisJob::assign(const STATAnalysisJob & aj) { boot_rep_prop = aj.boot_rep_prop; n_boot_rep = aj.n_boot_rep; + ss_index_name = aj.ss_index_name; hss_ec_value = aj.hss_ec_value; rank_corr_flag = aj.rank_corr_flag; vif_flag = aj.vif_flag; @@ -628,6 +631,9 @@ void STATAnalysisJob::dump(ostream & out, int depth) const { out << prefix << "boot_seed = " << boot_seed << "\n"; + out << prefix << "ss_index_name = " + << ss_index_name << "\n"; + out << prefix << "hss_ec_value = " << hss_ec_value << "\n"; @@ -1606,6 +1612,10 @@ void STATAnalysisJob::parse_job_command(const char *jobstring) { set_boot_seed(jc_array[i+1].c_str()); i++; } + else if(jc_array[i] == "-ss_index_name") { + ss_index_name = jc_array[i+1].c_str(); + i++; + } else if(jc_array[i] == "-hss_ec_value") { hss_ec_value = atof(jc_array[i+1].c_str()); i++; @@ -2776,6 +2786,15 @@ ConcatString STATAnalysisJob::get_jobstring() const { } } + // Jobs which compute the skill score index + if(job_type == stat_job_go_index || + job_type == stat_job_cbs_index || + job_type == stat_job_ss_index) { + + // ss_index_name + js << "-ss_index_name " << ss_index_name << " "; + } + // Jobs which write MCTC or MCTS output if(!is_bad_data(hss_ec_value) && (out_line_type.has(stat_mctc_str) || diff --git a/met/src/libcode/vx_analysis_util/stat_job.h b/met/src/libcode/vx_analysis_util/stat_job.h index 117fd58d6f..bef4f68f1e 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.h +++ b/met/src/libcode/vx_analysis_util/stat_job.h @@ -324,6 +324,11 @@ class STATAnalysisJob { char *boot_rng; char *boot_seed; + // + // Name of the skill score index being computed + // + ConcatString ss_index_name; + // // MCTS HSS Expected Correct rate // From 41a3d056a03086d22d86bfc2f7254abe930a831d Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 21:47:08 -0600 Subject: [PATCH 36/60] Per #1788, change from name to ss_index_name variable to be a bit more descriptive in the variable names. --- met/src/libcode/vx_statistics/met_stats.h | 4 ++-- .../stat_analysis/skill_score_index_job.cc | 24 +++++++++---------- .../stat_analysis/skill_score_index_job.h | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/met/src/libcode/vx_statistics/met_stats.h b/met/src/libcode/vx_statistics/met_stats.h index 69824fa898..1c2cfc353b 100644 --- a/met/src/libcode/vx_statistics/met_stats.h +++ b/met/src/libcode/vx_statistics/met_stats.h @@ -672,8 +672,8 @@ inline double DMAPInfo::get_beta_value() const { return(beta_value); } struct SSIDXData { - // Index name - ConcatString name; + // Skill score index name + ConcatString ss_index_name; // Forecast and reference model names ConcatString fcst_model; diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.cc b/met/src/tools/core/stat_analysis/skill_score_index_job.cc index 548562e717..e46f917033 100644 --- a/met/src/tools/core/stat_analysis/skill_score_index_job.cc +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.cc @@ -82,7 +82,7 @@ void SSIndexJobInfo::init_from_scratch() { void SSIndexJobInfo::clear() { // Initialize - name.clear(); + ss_index_name.clear(); fcst_model.clear(); ref_model.clear(); n_term = 0; @@ -103,7 +103,7 @@ void SSIndexJobInfo::clear() { void SSIndexJobInfo::assign(const SSIndexJobInfo &c) { - name = c.name; + ss_index_name = c.ss_index_name; fcst_model = c.fcst_model; ref_model = c.ref_model; n_term = c.n_term; @@ -232,7 +232,7 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { SSIDXData data; mlog << Debug(3) - << "Computing " << name << " for " << init_time.n() + << "Computing " << ss_index_name << " for " << init_time.n() << " initialization time(s): " << write_css(init_time) << "\n"; // Compute a skill score for each term @@ -286,7 +286,7 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { else ss_cs << ss; // Print debug info for each term - mlog << Debug(3) << name << " Term " << i+1 + mlog << Debug(3) << ss_index_name << " Term " << i+1 << ": " << term_cs << ", " << n_fcst_lines[i] << " fcst " << fcst_job[i].line_type[0] << "->" @@ -321,12 +321,12 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { else ss_avg = ss_sum/weight_sum; // Store the output data - data.name = name; - data.fcst_model = fcst_model; - data.ref_model = ref_model; - data.init_time = init_time; - data.n_term = n_term; - data.n_vld = n_vld; + data.ss_index_name = ss_index_name; + data.fcst_model = fcst_model; + data.ref_model = ref_model; + data.init_time = init_time; + data.n_term = n_term; + data.n_vld = n_vld; if(is_bad_data(ss_avg) || is_eq(ss_avg, 1.0)) { data.ss_index = bad_data_double; } @@ -335,8 +335,8 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { } mlog << Debug(3) - << name << " Weighted Average = " << ss_avg << "\n" - << name << " Skill Score Value = " << data.ss_index << "\n"; + << ss_index_name << " Weighted Average = " << ss_avg << "\n" + << ss_index_name << " Skill Score Value = " << data.ss_index << "\n"; return(data); } diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.h b/met/src/tools/core/stat_analysis/skill_score_index_job.h index c1d52bfaf3..9e14341906 100644 --- a/met/src/tools/core/stat_analysis/skill_score_index_job.h +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.h @@ -60,7 +60,7 @@ class SSIndexJobInfo { SSIndexJobInfo & operator=(const SSIndexJobInfo &); // Index name - ConcatString name; + ConcatString ss_index_name; // Forecast and reference model names ConcatString fcst_model, ref_model; From cde0be0950949af7a7d69121cfe0d82b9dd4d2cf Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 21:48:27 -0600 Subject: [PATCH 37/60] Per #1788, since ss_index_name is in the StatJob class we no longer need to pass it around as an argument. --- met/src/tools/core/stat_analysis/aggr_stat_line.cc | 5 ++--- met/src/tools/core/stat_analysis/aggr_stat_line.h | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.cc b/met/src/tools/core/stat_analysis/aggr_stat_line.cc index 26994c9114..0cd52dd43c 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -3397,7 +3397,6 @@ void aggr_time_series_lines(LineDataFile &f, STATAnalysisJob &job, //////////////////////////////////////////////////////////////////////// void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, - const ConcatString &name, map &m, int &n_in, int &n_out) { STATLine line; @@ -3408,7 +3407,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, // // Store the index name // - cur.job_info.name = name; + cur.job_info.ss_index_name = job.ss_index_name; // // Check that the -model option has been supplied exactly 2 times. @@ -3427,7 +3426,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, } mlog << Debug(3) - << "Computing " << cur.job_info.name << " for forecast model (" + << "Computing " << cur.job_info.ss_index_name << " for forecast model (" << cur.job_info.fcst_model << ") versus reference model (" << cur.job_info.ref_model << ").\n"; diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.h b/met/src/tools/core/stat_analysis/aggr_stat_line.h index d07f9a7d87..9cf72aeb2f 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.h +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.h @@ -295,7 +295,6 @@ extern void aggr_time_series_lines( extern void aggr_ss_index( LineDataFile &, STATAnalysisJob &, - const ConcatString &, map &, int &, int &); From 5354d3e4a4f2fde4cadd704988dd3c0fdd3c4600 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 25 Aug 2021 21:50:28 -0600 Subject: [PATCH 38/60] Per #1788, simply the skill score index job handling logic a bit. We only need to update the default job once, not twice.: --- .../tools/core/stat_analysis/stat_analysis.cc | 60 +++---------------- .../core/stat_analysis/stat_analysis_job.cc | 17 ++---- 2 files changed, 12 insertions(+), 65 deletions(-) diff --git a/met/src/tools/core/stat_analysis/stat_analysis.cc b/met/src/tools/core/stat_analysis/stat_analysis.cc index 04914f9ec9..935d926e67 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis.cc @@ -479,6 +479,7 @@ void process_search_dirs() { // // Parse the Skill Score Index config file into the search job. // + ss_index_job.set_job_type(default_job.job_type); set_job_from_config(ss_index_conf, ss_index_job); // @@ -635,9 +636,8 @@ return; //////////////////////////////////////////////////////////////////////// void process_job(const char * jobstring, int n_job) { - STATAnalysisJob job, ss_index_job; + STATAnalysisJob job; ConcatString full_jobstring; - MetConfig ss_index_conf; mlog << Debug(4) << "process_job(jobstring): " << jobstring << "\n"; @@ -651,58 +651,12 @@ void process_job(const char * jobstring, int n_job) { job = default_job; // - // Parse the job command line options - // - if(jobstring != command_line_job_options) { - mlog << Debug(4) - << "\nAmending Job " << n_job << " with options: \"" - << jobstring << "\"\n"; - job.parse_job_command(jobstring); - } - - // - // Special processing for the GO Index and CBS Index jobs. - // - if(default_job.job_type == stat_job_go_index || - default_job.job_type == stat_job_cbs_index) { - - ConcatString config_file = - (default_job.job_type == stat_job_go_index ? - replace_path(go_index_config_file) : - replace_path(cbs_index_config_file)); - - mlog << Debug(1) << "Skill Score Index Config File: " - << config_file << "\n"; - - // - // Read the config files for the constants and the skill score index. - // - ss_index_conf.read(replace_path(config_const_filename).c_str()); - ss_index_conf.read(config_file.c_str()); - - // - // Parse the Skill Score Index config file into the search job. - // - set_job_from_config(ss_index_conf, ss_index_job); - - // - // Amend the current job with Skill Score Index filtering criteria. - // - mlog << Debug(4) - << "\nAmending Job " << n_job << " with Skill Score Index configuration file: " - << config_file << "\n"; - job.parse_job_command(ss_index_job.get_jobstring().c_str()); - } - - // - // Amend the current job using any command line options + // Override with any command line options // - if(jobstring != command_line_job_options) { - mlog << Debug(4) - << "\nAmending Job " << n_job << " with command line options: \"" - << command_line_job_options << "\"\n"; - job.parse_job_command(command_line_job_options.c_str()); - } + mlog << Debug(4) + << "\nAmending Job " << n_job << " with command line options: \"" + << command_line_job_options << "\"\n"; + job.parse_job_command(command_line_job_options.c_str()); // // Get the full jobstring diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index 2580199d87..7854ca8f3c 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -121,6 +121,7 @@ void set_job_from_config(MetConfig &c, STATAnalysisJob &job) { job.set_boot_rng(boot_info.rng.c_str()); job.set_boot_seed(boot_info.seed.c_str()); + job.ss_index_name = c.lookup_string(conf_key_ss_index_name); job.hss_ec_value = c.lookup_double(conf_key_hss_ec_value); job.rank_corr_flag = (int) c.lookup_bool(conf_key_rank_corr_flag); job.vif_flag = (int) c.lookup_bool(conf_key_vif_flag); @@ -3979,8 +3980,8 @@ void write_job_ss_index(STATAnalysisJob &job, // Set FCST/OBS_VAR = skill score index type // Set FCST/OBS_UNITS = FCST/OBS_LEV = NA // - shc.set_fcst_var(it->second.job_info.name); - shc.set_obs_var(it->second.job_info.name); + shc.set_fcst_var(it->second.job_info.ss_index_name); + shc.set_obs_var(it->second.job_info.ss_index_name); shc.set_fcst_units(na_str); shc.set_obs_units(na_str); shc.set_fcst_lev(na_str); @@ -4010,7 +4011,7 @@ void write_job_ss_index(STATAnalysisJob &job, job.stat_row++, n_header_columns); } else { - at.set_entry(r, c++, ssidx_data.name); + at.set_entry(r, c++, it->second.job_info.ss_index_name); write_case_cols(it->first, at, r, c); write_ssidx_cols(ssidx_data, at, r++, c); } @@ -4041,7 +4042,6 @@ void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, STATAnalysisJob &job, int &n_in, int &n_out, ofstream *sa_out) { map ssidx_map; - ConcatString name; AsciiTable out_at; // @@ -4049,17 +4049,10 @@ void do_job_ss_index(const ConcatString &jobstring, LineDataFile &f, // job.out_line_type.add(stat_ssidx_str); - // - // Determine the job type - // - if(job.job_type == stat_job_go_index) name = "GO_INDEX"; - else if(job.job_type == stat_job_cbs_index) name = "CBS_INDEX"; - else name = "SS_INDEX"; - // // Compute the Skill Score Index // - aggr_ss_index(f, job, name, ssidx_map, n_in, n_out); + aggr_ss_index(f, job, ssidx_map, n_in, n_out); // // Write the output From bbfd838debf9eeffc35f96b77f04a081ef2cd189 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 26 Aug 2021 08:33:59 -0600 Subject: [PATCH 39/60] Revert "Per #1788, since ss_index_name is in the StatJob class we no longer need to pass it around as an argument." This reverts commit cde0be0950949af7a7d69121cfe0d82b9dd4d2cf. --- met/src/tools/core/stat_analysis/aggr_stat_line.cc | 5 +++-- met/src/tools/core/stat_analysis/aggr_stat_line.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.cc b/met/src/tools/core/stat_analysis/aggr_stat_line.cc index 0cd52dd43c..26994c9114 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -3397,6 +3397,7 @@ void aggr_time_series_lines(LineDataFile &f, STATAnalysisJob &job, //////////////////////////////////////////////////////////////////////// void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, + const ConcatString &name, map &m, int &n_in, int &n_out) { STATLine line; @@ -3407,7 +3408,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, // // Store the index name // - cur.job_info.ss_index_name = job.ss_index_name; + cur.job_info.name = name; // // Check that the -model option has been supplied exactly 2 times. @@ -3426,7 +3427,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, } mlog << Debug(3) - << "Computing " << cur.job_info.ss_index_name << " for forecast model (" + << "Computing " << cur.job_info.name << " for forecast model (" << cur.job_info.fcst_model << ") versus reference model (" << cur.job_info.ref_model << ").\n"; diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.h b/met/src/tools/core/stat_analysis/aggr_stat_line.h index 9cf72aeb2f..d07f9a7d87 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.h +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.h @@ -295,6 +295,7 @@ extern void aggr_time_series_lines( extern void aggr_ss_index( LineDataFile &, STATAnalysisJob &, + const ConcatString &, map &, int &, int &); From a83c3553ff015f5c22a5db0f934cb774a6e3f8b0 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 26 Aug 2021 08:37:36 -0600 Subject: [PATCH 40/60] Revert "Revert "Per #1788, since ss_index_name is in the StatJob class we no longer need to pass it around as an argument."" This reverts commit bbfd838debf9eeffc35f96b77f04a081ef2cd189. --- met/src/tools/core/stat_analysis/aggr_stat_line.cc | 5 ++--- met/src/tools/core/stat_analysis/aggr_stat_line.h | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.cc b/met/src/tools/core/stat_analysis/aggr_stat_line.cc index 26994c9114..0cd52dd43c 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -3397,7 +3397,6 @@ void aggr_time_series_lines(LineDataFile &f, STATAnalysisJob &job, //////////////////////////////////////////////////////////////////////// void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, - const ConcatString &name, map &m, int &n_in, int &n_out) { STATLine line; @@ -3408,7 +3407,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, // // Store the index name // - cur.job_info.name = name; + cur.job_info.ss_index_name = job.ss_index_name; // // Check that the -model option has been supplied exactly 2 times. @@ -3427,7 +3426,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, } mlog << Debug(3) - << "Computing " << cur.job_info.name << " for forecast model (" + << "Computing " << cur.job_info.ss_index_name << " for forecast model (" << cur.job_info.fcst_model << ") versus reference model (" << cur.job_info.ref_model << ").\n"; diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.h b/met/src/tools/core/stat_analysis/aggr_stat_line.h index d07f9a7d87..9cf72aeb2f 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.h +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.h @@ -295,7 +295,6 @@ extern void aggr_time_series_lines( extern void aggr_ss_index( LineDataFile &, STATAnalysisJob &, - const ConcatString &, map &, int &, int &); From b2210702f8e0112628152580703c8b99e7debb78 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 26 Aug 2021 08:49:22 -0600 Subject: [PATCH 41/60] Per #1788, I'd pruned too much code from stat_analysis.cc which caused the processing of config file jobs to fail. Fixing that here. --- met/src/tools/core/stat_analysis/stat_analysis.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/met/src/tools/core/stat_analysis/stat_analysis.cc b/met/src/tools/core/stat_analysis/stat_analysis.cc index 935d926e67..7d987d7c42 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis.cc @@ -650,6 +650,16 @@ void process_job(const char * jobstring, int n_job) { << default_job.get_jobstring() << "\"\n"; job = default_job; + // + // Parse the job command line options + // + if(jobstring != command_line_job_options) { + mlog << Debug(4) + << "\nAmending Job " << n_job << " with options: \"" + << jobstring << "\"\n"; + job.parse_job_command(jobstring); + } + // // Override with any command line options // From eb5b147d2dfa8de02eab76f775188f80c409aeb7 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 26 Aug 2021 11:49:27 -0600 Subject: [PATCH 42/60] Per #1788, switch Z0 to L0 to match the sample data for computing CBS Index. --- met/data/config/STATAnalysisConfig_CBS_Index | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/met/data/config/STATAnalysisConfig_CBS_Index b/met/data/config/STATAnalysisConfig_CBS_Index index 745c5efefa..dbb5b9cf65 100644 --- a/met/data/config/STATAnalysisConfig_CBS_Index +++ b/met/data/config/STATAnalysisConfig_CBS_Index @@ -57,12 +57,12 @@ fcst_var = [ "PRMSL", "PRMSL", "PRMSL", "PRMSL", "PRMSL", ]; obs_var = []; -fcst_lev = [ "Z0", "Z0", "Z0", "Z0", "Z0", +fcst_lev = [ "L0", "L0", "L0", "L0", "L0", "P500", "P500", "P500", "P500", "P500", "P250", "P250", "P250", "P250", "P250", "P850", "P850", "P850", "P850", "P850", "P250", "P250", "P250", "P250", "P250", - "Z0", "Z0", "Z0", "Z0", "Z0", + "L0", "L0", "L0", "L0", "L0", "P500", "P500", "P500", "P500", "P500", "P250", "P250", "P250", "P250", "P250" ]; From 22ed9c68eb88352c7f763ce588b2b58fb0244832 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 26 Aug 2021 12:49:56 -0600 Subject: [PATCH 43/60] Per #1788, make it so that cbs_index and go_index work from config file. --- .../tools/core/stat_analysis/stat_analysis.cc | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/met/src/tools/core/stat_analysis/stat_analysis.cc b/met/src/tools/core/stat_analysis/stat_analysis.cc index 7d987d7c42..4e891d49ee 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis.cc @@ -660,6 +660,41 @@ void process_job(const char * jobstring, int n_job) { job.parse_job_command(jobstring); } + // + // Special processing for the GO Index and CBS Index jobs. + // + if(job.job_type == stat_job_go_index || + job.job_type == stat_job_cbs_index) { + + MetConfig ss_index_conf; + STATAnalysisJob ss_index_job; + + ConcatString config_file = + (job.job_type == stat_job_go_index ? + replace_path(go_index_config_file) : + replace_path(cbs_index_config_file)); + + // + // Read the config files for the constants and the skill score index. + // + ss_index_conf.read(replace_path(config_const_filename).c_str()); + ss_index_conf.read(config_file.c_str()); + + // + // Parse the Skill Score Index config file into the search job. + // + ss_index_job.set_job_type(job.job_type); + set_job_from_config(ss_index_conf, ss_index_job); + + // + // Amend the current job with Skill Score Index filtering criteria. + // + mlog << Debug(4) + << "\nAmending Job " << n_job << " with Skill Score Index configuration file: " + << config_file << "\n"; + job.parse_job_command(ss_index_job.get_jobstring().c_str()); + } + // // Override with any command line options // From bf7b1f42401de49757588b0006f92e0325ff3a2c Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 26 Aug 2021 17:07:01 -0600 Subject: [PATCH 44/60] Per #1788, update the documentation for the skill score index changes. --- met/docs/Users_Guide/point-stat.rst | 2 +- met/docs/Users_Guide/stat-analysis.rst | 160 +++++++++++++++++++++++-- met/docs/Users_Guide/wavelet-stat.rst | 2 +- 3 files changed, 153 insertions(+), 11 deletions(-) diff --git a/met/docs/Users_Guide/point-stat.rst b/met/docs/Users_Guide/point-stat.rst index 97b93b4b40..c37c5b06d7 100644 --- a/met/docs/Users_Guide/point-stat.rst +++ b/met/docs/Users_Guide/point-stat.rst @@ -498,7 +498,7 @@ The first set of header columns are common to all of the output files generated .. _table_PS_header_info_point-stat_out: -.. list-table:: Header information for each file point-stat outputs. +.. list-table:: Common STAT header columns. :widths: auto :header-rows: 2 diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index 2267f7ba03..4df775de56 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -52,10 +52,49 @@ When aggregating the matched pair line type (MPR) and computing an output contin .. _StA_Skill-Score-Index: -Skill Score Index, including GO Index -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Skill Score Index +~~~~~~~~~~~~~~~~~ + +The Stat-Analysis "ss_index", "go_index", and "cbs_index" jobs calculate skill score indices by weighting scores for different meteorological fields at different pressure levels and for different lead times. Pre-defined configuration files are provided for the GO Index and CBS Index which are special cases of the highly configurable skill score index job. + +In general, a skill score index is computed over several terms and the number and definition of those terms is configurable. It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases. When configuring a skill score index job, the following requirements apply: + +1. Exactly two models names must be chosen. The first is interpreted as the forecast model and the second is the reference model, against which the performance of the forecast should be measured. Specify this with the "model" configuration file entry or using the "-model" job command option. + +2. The forecast variable name, level, lead time, line type, column, and weight options must be specified. If the value remains contant for all the terms, set its to an array of length one. If the value changes for at least one term, specify an array entry for each term. Specify these with the "fcst_var", "fcst_lev", "lead_time", "line_type", "column", and "weight" configuration file entries, respectively, or use the corresponding job command options. + +3. While these line types are required, additional options may be provided for each term, including the observation type ("obtype"), verification region ("vx_mask"), and interpolation method ("interp_mthd"). Specify each as single value or provide a value for each term. + +4. Only the SL1L2 and CTC input line types are supported, and the input Point-Stat and/or Grid-Stat output must contain these line types. + +5. For the SL1L2 line type, set the "column" entry to the CNT output column that contains the statistic of interest (e.g. RMSE for root-mean-squared-error). Note, only those continuous statistics that are derivable from SL1L2 lines can be used. + +6. For the CTC line type, set the "column" entry to the CTS output column that contains the statistic of intereest (.e.g. PODY for probability of detecting yes). Note, consider specifying the "fcst_thresh" for the CTC line type. + +For each term, all matching SL1L2 (or CTC) input lines are aggregated separtely for the forecast and reference models. The requested statistic ("column") is derived from the aggregated partial sums or counts. For each term, a skill score is defined as: + +.. math:: ss = 1.0 - \frac{s_{fcst}^2}{s_{ref}^2} + +Where :math:`s_{fcst}` and :math:`s_{ref}` are the aggregated forecast and reference statistics, respectively. Next, a weighted average is computed from the skill scores for each term: -The Stat-Analysis "ss_index" and "go_index" jobs calculate the skill score indices by weighting scores for different meteorological fields at different pressure levels and for different lead times. The GO Index is a special case of the Skill Score index for which a specific configuration file is provided. The GO index is a weighted average of the RMSE values for wind speed, dew point temperature, temperature, height, and pressure at several levels in the atmosphere. The variables, levels, and lead times included in the index are shown in :numref:`compute_GO_Index` and are defined by a default Stat-Analysis configuration file. The partial sums (SL1L2 lines in the STAT output) for each of these variables at each level and lead time must have been computed in a previous step. The Stat-Analysis tool then uses the weights in :numref:`compute_GO_Index` to compute values for the GO Index. For a general skill score index, the user can specify the weights and variables to use in the calculations in a Stat-Analysis configuration file and run the ss_index job type. + .. math:: ss_{avg} = \frac{1}{n} \sum_{i=1}^{n} w_i * ss_i + +Where, :math:`w_i` and :math:`ss_i` are the weight and skill score for each term and :math:`n` is the number of terms. Finally, the skill score index is computed as: + +.. math:: index = \sqrt{\frac{1.0}{1.0-ss_{avg}}} + +A value greater than 1.0 indicates that the forecast model outperforms the reference, while a value less than 1.0 indicates that the reference outperforms the forecast. + +The default skill score index name (SS_INDEX) can be overridden using the "ss_index_name" option in the configuration file. The pre-defined configuration files for the GO Index and CBS Index use "GO_INDEX" and "CBS_INDEX", respectively. + +When running a skill score index job using the "-out_stat" job command option, a .stat output file is written containing the skill score index (SSIDX) output line type. If the "-by" job command option is specified, the skill score index will be computed separately for each unique combination of values found in the column(s) specified. For example, "-by FCST_INIT_BEG,VX_MASK" runs the job separately for each combination of model initialization time and verification region found in the input. Note that increasing the Stat-Analysis verbosity level (-v 3) on the command line prints detailed information about each skill score index term. + +.. _StA_Go-Index: + +GO Index +~~~~~~~~ + +The "go_index" job is a special case of the "ss_index" job, described in :numref:`StA_Skill-Score-Index`. The GO Index is a weighted average of 48 skill scores of RMSE statistics for wind speed, dew point temperature, temperature, height, and pressure at several levels in the atmosphere. The variables, levels, and lead times included in the index are shown in :numref:`compute_GO_Index` and are defined by the default STATAnalysisConfig_GO_Index configuration file. The partial sums (SL1L2 lines in the STAT output) for each of these variables at each level and lead time must have been computed in a previous step. The Stat-Analysis tool then uses the weights in :numref:`compute_GO_Index` to compute values for the GO Index. .. _compute_GO_Index: @@ -148,6 +187,50 @@ The Stat-Analysis "ss_index" and "go_index" jobs calculate the skill score indic - 4 - 2 +.. _StA_CBS-Index: + +CBS Index +~~~~~~~~~ + +The "cbs_index" job is a special case of the "ss_index" job, described in :numref:`StA_Skill-Score-Index`. The CBS Index is a weighted average of 40 skill scores of RMSE statistics for mean sea level pressure, height, and wind speed at multiple levels computed over the northern hemisphere, southern hemisphere and the tropics. The variables, levels, lead times, and regions included in the index are shown in :numref:`compute_CBS_Index` and are defined by a default Stat-Analysis configuration file. The partial sums (SL1L2 lines in the STAT output) for each of these variables for each level, lead time, and masking region must have been computed in a previous step. The Stat-Analysis tool then uses the weights in :numref:`compute_CBS_Index` to compute values for the CBS Index. + +.. _compute_CBS_Index: + +.. list-table:: Variables, levels, and weights used to compute the CBS Index for 24, 48, 72, 96 and 120 hour lead times. + :widths: auto + :header-rows: 2 + + * - Variable + - Level + - Weights by Region + - + - + * - + - + - North Hem + - Tropics + - South Hem + * - Pressure + - Mean sea level + - 6.4 + - x + - 3.2 + * - Height + - 500 hPa + - 2.4 + - x + - 1.2 + * - Wind speed + - 250 hPa + - 2.4 + - 1.2 + - 1.2 + * - + - 850 hPa + - x + - 2.0 + - x + Ramp Events ~~~~~~~~~~~ @@ -156,7 +239,7 @@ The Stat-Analysis "ramp" job identifies ramp events (large increases or decrease Wind Direction Statistics ~~~~~~~~~~~~~~~~~~~~~~~~~ -The Stat-Analysis "aggregate_stat" job can read vector partial sums and derive wind direction error statistics (WDIR). The vector partial sums (VL1L2 or VAL1L2) or matched pairs (MPR) for the UGRD and VGRD must have been computed in a previous step, i.e. by Point-Stat or Grid-Stat tools. This job computes an average forecast wind direction and an average observed wind direction along with their difference. The output is in degrees. In Point-Stat and Grid-Stat, the UGRD and VGRD can be verified using thresholds on their values or on the calculated wind speed. If thresholds have been applied, the wind direction statistics are calculated for each threshold. +The Stat-Analysis "aggregate_stat" job can read vector partial sums and derive wind direction error statistics (WDIR). The vector partial sums (VL1L2 or VAL1L2) or matched pairs (MPR) for the UGRD and VGRD must have been computed in a previous step, i.e. by Point-Stat or Grid-Stat tools. This job computes an average forecast wind direction and an average observed wind direction along with their difference. The output is in degrees. In Point-Stat and Grid-Stat, the UGRD and VGRD can be verified using thresholds on their values or on the calculated wind speed. If thresholds have been applied, the wind direction statistics are calculated for each threshold. The first step in verifying wind direction is running the Grid-Stat and/or Point-Stat tools to verify each forecast of interest and generate the VL1L2 or MPR line(s). When running these tools, please note: @@ -476,7 +559,10 @@ All possible tasks for **job_name** are listed in :numref:`Des_components_STAT_a - Calculates a user-defined Skill Score index as described in section :numref:`StA_Skill-Score-Index`. - \-model forecast :raw-html:`
` \-model reference * - go_index - - Calculates the GO Index as described in section :numref:`StA_Skill-Score-Index`. + - Calculates the GO Index as described in section :numref:`StA_GO-Index`. + - \-model forecast :raw-html:`
` \-model reference + * - cbs_index + - Calculates the CBS Index as described in section :numref:`StA_CBS-Index`. - \-model forecast :raw-html:`
` \-model reference * - ramp - Defines a ramp event on a time-series of forecast and observed values. The amount of change from one time to the next is computed for forecast and observed values. Those changes are thresholded to define events which are used to populate a 2x2 contingency table. @@ -694,13 +780,69 @@ This job is similar to the "**aggregate**" job listed above, however the format * - MPR - FHO, CTC, CTS, MCTC, MCTS, PCT, PSTD, PJC, or PRC (must specify "**-out_fcst_thresh**" and "**-out_obs_thresh**" arguments) -**Job: ss_index** +**Job: ss_index, go_index, cbs_index** + +While the inputs for the "ss_index", "go_index", and "cbs_index" jobs may vary, the output is the same. By default, the job output is written to the screen or to a "-out" file, if specified. If the "-out_stat" job command option is specified, a STAT output file is written containing the skill score index (SSIDX) output line type. + +The SSIDX line type consists of the common STAT header columns described in :numref:`table_PS_header_info_point-stat_out` followed by the columns described below. In general, when multiple input header strings are encountered, the output is reported as a comma-separated list of the unique values. The "-set_hdr" job command option can be used to override any of the output header strings (e.g. "-set_hdr VX_MASK MANY" sets the output VX_MASK column to "MANY"). Special logic applied to some of the STAT header columns are also described below. -The output from this job consists of three lines, the first two of which contain "**JOB_LIST**" and "**COL_NAME**", as described above. The third line contains "**SS_INDEX**" followed by a colon and then the value computed for the user-defined Skill Score Index. +.. _table_SA_format_info_SSIDX: -**Job: go_index** +.. list-table:: Format information for the SSIDX (Skill Score Index) output line type. + :widths: auto + :header-rows: 2 -The output from this job consists of three lines, the first two of which contain "**JOB_LIST**" and"**COL_NAME**", as described above. The third line contains "**GO_INDEX**" followed by a colon and then the value computed for the GO Index. + * - SSIDX OUTPUT FORMAT + - + - + * - Column Number + - SSIDX Column Name + - Description + * - 4 + - FCST_LEAD + - Maximum input forecast lead time + * - 5 + - FCST_VALID_BEG + - Minimum input forecast valid start time + * - 6 + - FCST_VALID_END + - Maximum input forecast valid end time + * - 7 + - OBS_LEAD + - Maximum input observation lead time + * - 8 + - OBS_VALID_BEG + - Minimum input observation valid start time + * - 9 + - OBS_VALID_END + - Maximum input observation valid end time + * - 10 + - FCST_VAR + - Skill score index name from the "ss_index_name" option + * - 11 + - OBS_VAR + - Skill score index name from the "ss_index_name" option + * - 24 + - SSIDX + - Skill score index line type + * - 25 + - FCST_MODEL + - Forecast model name + * - 26 + - REF_MODEL + - Reference model name + * - 27 + - N_INIT + - Number of unique input model initialization times + * - 28 + - N_TERM + - Number of skill score index terms + * - 29 + - N_VLD + - Number of terms for which a valid skill score was computed + * - 30 + - SS_INDEX + - Skill score index value **Job: ramp** diff --git a/met/docs/Users_Guide/wavelet-stat.rst b/met/docs/Users_Guide/wavelet-stat.rst index e449a513e3..2b9a39291b 100644 --- a/met/docs/Users_Guide/wavelet-stat.rst +++ b/met/docs/Users_Guide/wavelet-stat.rst @@ -323,7 +323,7 @@ The output ASCII files are named similarly: wavelet_stat_PREFIX_HHMMSSL_YYYYMMDD_HHMMSSV_TYPE.txt where TYPE is isc to indicate that this is an intensity-scale line type. -The format of the STAT and ASCII output of the Wavelet-Stat tool is similar to the format of the STAT and ASCII output of the Point-Stat tool. Please refer to the tables in :numref:`point_stat-output` for a description of the common output for STAT files types. The information contained in the STAT and isc files are identical. However, for consistency with the STAT files produced by other tools, the STAT file will only have column headers for the first 21 fields. The isc file contains all headers. The format of the ISC line type is explained in the following table. +The format of the STAT and ASCII output of the Wavelet-Stat tool is similar to the format of the STAT and ASCII output of the Point-Stat tool. Please refer to the tables in :numref:`point_stat-output` for a description of the common output for STAT files types. The information contained in the STAT and isc files are identical. However, for consistency with the STAT files produced by other tools, the STAT file will only have names for the header columns. The isc file contains names for all columns. The format of the ISC line type is explained in the following table. .. _table_WS_header_info_ws_outputs: From ecc6463c5dba749bdcabff02149cc4ebf85a6bf9 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 26 Aug 2021 17:08:56 -0600 Subject: [PATCH 45/60] Per #1788, just move some variable definitions further down since they're only used sometimes. --- met/src/tools/core/stat_analysis/stat_analysis.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/met/src/tools/core/stat_analysis/stat_analysis.cc b/met/src/tools/core/stat_analysis/stat_analysis.cc index 4e891d49ee..de7a7bf076 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis.cc @@ -434,8 +434,6 @@ void set_out_file(const char *path) { void process_search_dirs() { int n, i, j, max_len, n_read, n_keep; - MetConfig ss_index_conf; - STATAnalysisJob ss_index_job; // // Initialize @@ -462,6 +460,9 @@ void process_search_dirs() { if(default_job.job_type == stat_job_go_index || default_job.job_type == stat_job_cbs_index) { + MetConfig ss_index_conf; + STATAnalysisJob ss_index_job; + ConcatString config_file = (default_job.job_type == stat_job_go_index ? replace_path(go_index_config_file) : From 9b779daba5dc3bb93a36e94bf11349fd53fff8a7 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 26 Aug 2021 17:15:40 -0600 Subject: [PATCH 46/60] Per #1788, make job output description into a numbered section. --- met/docs/Users_Guide/stat-analysis.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index 4df775de56..73a36e1cb2 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -780,7 +780,8 @@ This job is similar to the "**aggregate**" job listed above, however the format * - MPR - FHO, CTC, CTS, MCTC, MCTS, PCT, PSTD, PJC, or PRC (must specify "**-out_fcst_thresh**" and "**-out_obs_thresh**" arguments) -**Job: ss_index, go_index, cbs_index** +Job: ss_index, go_index, cbs_index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ While the inputs for the "ss_index", "go_index", and "cbs_index" jobs may vary, the output is the same. By default, the job output is written to the screen or to a "-out" file, if specified. If the "-out_stat" job command option is specified, a STAT output file is written containing the skill score index (SSIDX) output line type. @@ -844,7 +845,8 @@ The SSIDX line type consists of the common STAT header columns described in :num - SS_INDEX - Skill score index value -**Job: ramp** +Job: ramp +^^^^^^^^^ The ramp job operates on a time-series of forecast and observed values and is analogous to the RIRW (Rapid Intensification and Weakening) job described in :numref:`tc_stat-output`. The amount of change from one time to the next is computed for forecast and observed values. Those changes are thresholded to define events which are used to populate a 2x2 contingency table. From f3136602ad2c03958c95ffe28d367c564c5a0b96 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 26 Aug 2021 17:21:51 -0600 Subject: [PATCH 47/60] Per #1788, little tweak. --- met/docs/Users_Guide/stat-analysis.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index 73a36e1cb2..1143025811 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -742,7 +742,6 @@ This job produces summary statistics for the column name and line type specified * - 22 - WMO Weighted Mean value - Job: aggregate ^^^^^^^^^^^^^^ From dc0fa3ea8adf5318cfb3aa40aab929f858b60a5b Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 27 Aug 2021 09:58:13 -0600 Subject: [PATCH 48/60] Per #1788, update the STATAnalysisConfig files to add ss_index_name and hss_ec_value, as needed. --- met/scripts/config/STATAnalysisConfig | 1 + test/config/STATAnalysisConfig_APCP_HIRA | 2 ++ test/config/STATAnalysisConfig_climo | 2 ++ test/config/STATAnalysisConfig_filter_times | 2 ++ test/config/STATAnalysisConfig_grid_stat | 2 ++ test/config/STATAnalysisConfig_point_stat | 2 ++ test/config/STATAnalysisConfig_ramps | 2 ++ 7 files changed, 13 insertions(+) diff --git a/met/scripts/config/STATAnalysisConfig b/met/scripts/config/STATAnalysisConfig index 6ff87047e5..5f81208f5a 100644 --- a/met/scripts/config/STATAnalysisConfig +++ b/met/scripts/config/STATAnalysisConfig @@ -104,6 +104,7 @@ boot = { //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "SS_INDEX"; hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; diff --git a/test/config/STATAnalysisConfig_APCP_HIRA b/test/config/STATAnalysisConfig_APCP_HIRA index 22faf3ae7a..d7a666674f 100644 --- a/test/config/STATAnalysisConfig_APCP_HIRA +++ b/test/config/STATAnalysisConfig_APCP_HIRA @@ -98,6 +98,8 @@ boot = { //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "SS_INDEX"; +hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; tmp_dir = "/tmp"; diff --git a/test/config/STATAnalysisConfig_climo b/test/config/STATAnalysisConfig_climo index d316900db6..bd604acf3e 100644 --- a/test/config/STATAnalysisConfig_climo +++ b/test/config/STATAnalysisConfig_climo @@ -104,6 +104,8 @@ wmo_fisher_stats = [ "CNT:PR_CORR", "CNT:SP_CORR", //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "SS_INDEX"; +hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; tmp_dir = "/tmp"; diff --git a/test/config/STATAnalysisConfig_filter_times b/test/config/STATAnalysisConfig_filter_times index 8a9d7afcb4..ced2004239 100644 --- a/test/config/STATAnalysisConfig_filter_times +++ b/test/config/STATAnalysisConfig_filter_times @@ -101,6 +101,8 @@ boot = { //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "SS_INDEX"; +hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; tmp_dir = "/tmp"; diff --git a/test/config/STATAnalysisConfig_grid_stat b/test/config/STATAnalysisConfig_grid_stat index 1453703dfc..b8e9fbb9ba 100644 --- a/test/config/STATAnalysisConfig_grid_stat +++ b/test/config/STATAnalysisConfig_grid_stat @@ -94,6 +94,8 @@ boot = { //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "SS_INDEX"; +hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; tmp_dir = "/tmp"; diff --git a/test/config/STATAnalysisConfig_point_stat b/test/config/STATAnalysisConfig_point_stat index d102f40825..c6dd05832f 100644 --- a/test/config/STATAnalysisConfig_point_stat +++ b/test/config/STATAnalysisConfig_point_stat @@ -123,6 +123,8 @@ boot = { //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "SS_INDEX"; +hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; tmp_dir = "/tmp"; diff --git a/test/config/STATAnalysisConfig_ramps b/test/config/STATAnalysisConfig_ramps index 5a5d06e424..cd7bec39a3 100644 --- a/test/config/STATAnalysisConfig_ramps +++ b/test/config/STATAnalysisConfig_ramps @@ -93,6 +93,8 @@ boot = { //////////////////////////////////////////////////////////////////////////////// +ss_index_name = "SS_INDEX"; +hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; tmp_dir = "/tmp"; From 51101253cbeb182afa2eb55309f653d4500c6257 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 27 Aug 2021 10:37:38 -0600 Subject: [PATCH 49/60] Per #1788, add new unit tests to exercise recent ss_index changes and update the Stat-Analysis documentation. --- met/docs/Users_Guide/stat-analysis.rst | 7 +- test/config/STATAnalysisConfig_SFC_SS_Index | 105 ++++++++++++++++++++ test/xml/unit_ref_config.xml | 37 +++++++ 3 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 test/config/STATAnalysisConfig_SFC_SS_Index diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index 1143025811..0ac8347724 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -512,10 +512,11 @@ ___________________ .. code-block:: none - column = []; - weight = []; + column = []; + weight = []; + ss_index_name = "SS_INDEX"; -The column and weight fields are used to define a skill score index. The computation of a single value will be computed from each column and weight value specified. The GO Index is a specific example of a skill score index. +The column, weight, and ss_index_name entries are used to define a skill score index. The column and weight arrays can either be set to a constant value of length one or specify a separate value for each term of the index. The ss_index_name entry is a string which defines the output name for the current skill score index configuration. ___________________ diff --git a/test/config/STATAnalysisConfig_SFC_SS_Index b/test/config/STATAnalysisConfig_SFC_SS_Index new file mode 100644 index 0000000000..1311321c91 --- /dev/null +++ b/test/config/STATAnalysisConfig_SFC_SS_Index @@ -0,0 +1,105 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// STAT-Analysis configuration file used to compute a Custom Skill Score Index. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Filtering input STAT lines by the contents of each column +// +model = [ "AFWAv3.4_Noahv3.3", "AFWAv3.4_Noahv2.7.1" ]; + +fcst_lead = [ "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48", + "12", "24", "36", "48" + ]; +obs_lead = []; + +fcst_valid_beg = ""; +fcst_valid_end = ""; +fcst_valid_inc = []; +fcst_valid_exc = []; +fcst_valid_hour = []; + +obs_valid_beg = ""; +obs_valid_end = ""; +obs_valid_inc = []; +obs_valid_exc = []; +obs_valid_hour = []; + +fcst_init_beg = ""; +fcst_init_end = ""; +fcst_init_inc = []; +fcst_init_exc = []; +fcst_init_hour = []; + +obs_init_beg = ""; +obs_init_end = ""; +obs_init_inc = []; +obs_init_exc = []; +obs_init_hour = []; + +fcst_var = [ "WIND", "WIND", "WIND", "WIND", + "DPT", "DPT", "DPT", "DPT", + "TMP", "TMP", "TMP", "TMP", + "PRMSL", "PRMSL", "PRMSL", "PRMSL" + ]; +obs_var = []; + +fcst_lev = [ "Z10", "Z10", "Z10", "Z10", + "Z2", "Z2", "Z2", "Z2", + "Z2", "Z2", "Z2", "Z2", + "Z0", "Z0", "Z0", "Z0" + ]; +obs_lev = []; + +obtype = []; + +vx_mask = []; + +interp_mthd = []; + +interp_pnts = []; + +fcst_thresh = []; +obs_thresh = []; +cov_thresh = []; + +alpha = []; + +line_type = [ "SL1L2" ]; + +column = [ "RMSE" ]; + +weight = [ 4.0, 3.0, 2.0, 1.0, + 4.0, 3.0, 2.0, 1.0, + 4.0, 3.0, 2.0, 1.0, + 4.0, 3.0, 2.0, 1.0 + ]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +out_alpha = 0.05; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +ss_index_name = "SFC_SS_INDEX"; +hss_ec_value = NA; +rank_corr_flag = FALSE; +vif_flag = FALSE; +tmp_dir = "/tmp"; +version = "V10.1.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/test/xml/unit_ref_config.xml b/test/xml/unit_ref_config.xml index 4e7e26113b..a18fffa811 100644 --- a/test/xml/unit_ref_config.xml +++ b/test/xml/unit_ref_config.xml @@ -953,4 +953,41 @@ + + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/ \ + -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/ \ + -job go_index \ + -by FCST_INIT_BEG,VX_MASK,OBTYPE -set_hdr DESC Noahv3.3_vs_v2.7.1 \ + -model AFWAv3.4_Noahv3.3,AFWAv3.4_Noahv2.7.1 \ + -out_stat &OUTPUT_DIR;/ref_config/stat_analysis/go_index_by_option.stat + + + &OUTPUT_DIR;/ref_config/stat_analysis/go_index_by_option.stat + + + + + + + + + &MET_BIN;/stat_analysis + \ + -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv3.3/ \ + -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/ \ + -job ss_index -config &CONFIG_DIR;/STATAnalysisConfig_SFC_SS_Index \ + -by FCST_INIT_BEG,VX_MASK,OBTYPE \ + -out_stat &OUTPUT_DIR;/ref_config/stat_analysis/sfc_ss_index_by_option.stat + + + &OUTPUT_DIR;/ref_config/stat_analysis/sfc_ss_index_by_option.stat + + + From fac383a1af185d71663e4ceb6ebf391e931f974c Mon Sep 17 00:00:00 2001 From: johnhg Date: Fri, 27 Aug 2021 11:41:52 -0600 Subject: [PATCH 50/60] Update met/docs/Users_Guide/stat-analysis.rst Co-authored-by: jprestop --- met/docs/Users_Guide/stat-analysis.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index 0ac8347724..c73cc06fa8 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -61,7 +61,7 @@ In general, a skill score index is computed over several terms and the number an 1. Exactly two models names must be chosen. The first is interpreted as the forecast model and the second is the reference model, against which the performance of the forecast should be measured. Specify this with the "model" configuration file entry or using the "-model" job command option. -2. The forecast variable name, level, lead time, line type, column, and weight options must be specified. If the value remains contant for all the terms, set its to an array of length one. If the value changes for at least one term, specify an array entry for each term. Specify these with the "fcst_var", "fcst_lev", "lead_time", "line_type", "column", and "weight" configuration file entries, respectively, or use the corresponding job command options. +2. The forecast variable name, level, lead time, line type, column, and weight options must be specified. If the value remains constant for all the terms, set it to an array of length one. If the value changes for at least one term, specify an array entry for each term. Specify these with the "fcst_var", "fcst_lev", "lead_time", "line_type", "column", and "weight" configuration file entries, respectively, or use the corresponding job command options. 3. While these line types are required, additional options may be provided for each term, including the observation type ("obtype"), verification region ("vx_mask"), and interpolation method ("interp_mthd"). Specify each as single value or provide a value for each term. From e486e2b670c61b1f91beffdc78702acd33e00ef0 Mon Sep 17 00:00:00 2001 From: johnhg Date: Fri, 27 Aug 2021 11:42:11 -0600 Subject: [PATCH 51/60] Update met/docs/Users_Guide/stat-analysis.rst Sorry for the typos! Thanks for catching them. Co-authored-by: jprestop --- met/docs/Users_Guide/stat-analysis.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index c73cc06fa8..23e4d76a7f 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -71,7 +71,7 @@ In general, a skill score index is computed over several terms and the number an 6. For the CTC line type, set the "column" entry to the CTS output column that contains the statistic of intereest (.e.g. PODY for probability of detecting yes). Note, consider specifying the "fcst_thresh" for the CTC line type. -For each term, all matching SL1L2 (or CTC) input lines are aggregated separtely for the forecast and reference models. The requested statistic ("column") is derived from the aggregated partial sums or counts. For each term, a skill score is defined as: +For each term, all matching SL1L2 (or CTC) input lines are aggregated separately for the forecast and reference models. The requested statistic ("column") is derived from the aggregated partial sums or counts. For each term, a skill score is defined as: .. math:: ss = 1.0 - \frac{s_{fcst}^2}{s_{ref}^2} From 58c74b91d966a964669962b08839f594cbf38c0b Mon Sep 17 00:00:00 2001 From: johnhg Date: Fri, 27 Aug 2021 11:42:25 -0600 Subject: [PATCH 52/60] Update met/docs/Users_Guide/stat-analysis.rst Co-authored-by: jprestop --- met/docs/Users_Guide/stat-analysis.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index 23e4d76a7f..933cc9d2ae 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -69,7 +69,7 @@ In general, a skill score index is computed over several terms and the number an 5. For the SL1L2 line type, set the "column" entry to the CNT output column that contains the statistic of interest (e.g. RMSE for root-mean-squared-error). Note, only those continuous statistics that are derivable from SL1L2 lines can be used. -6. For the CTC line type, set the "column" entry to the CTS output column that contains the statistic of intereest (.e.g. PODY for probability of detecting yes). Note, consider specifying the "fcst_thresh" for the CTC line type. +6. For the CTC line type, set the "column" entry to the CTS output column that contains the statistic of intereest (e.g. PODY for probability of detecting yes). Note, consider specifying the "fcst_thresh" for the CTC line type. For each term, all matching SL1L2 (or CTC) input lines are aggregated separately for the forecast and reference models. The requested statistic ("column") is derived from the aggregated partial sums or counts. For each term, a skill score is defined as: From 3f122591cc03321e88deb9714a88651b40873b36 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 27 Aug 2021 15:25:00 -0600 Subject: [PATCH 53/60] Per #1788, fix typo in error message. --- met/src/tools/core/stat_analysis/aggr_stat_line.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.cc b/met/src/tools/core/stat_analysis/aggr_stat_line.cc index 0cd52dd43c..9551cbbf49 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -3459,7 +3459,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, (job.line_type.n() != n_term && job.line_type.n() != 1) || (job.column.n() != n_term && job.column.n() != 1) || (job.weight.n() != n_term && job.weight.n() != 1)) { - mlog << Error << "\ncompute_ss_index() -> " + mlog << Error << "\naggr_ss_var() -> " << "each skill score index parameter must have the same length (" << n_term << ") or have length 1!\n" << "Check the \"-fcst_var\", \"-fcst_lev\", \"-fcst_lead\", " From 50d3e38e9b664f4933ba3b84df19b66fb1016f7a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 27 Aug 2021 15:58:00 -0600 Subject: [PATCH 54/60] Per #1788, fix a bug for ss_index... changing 'add' to 'set'. Also set the line_type seperately for each term, if requested. --- met/src/tools/core/stat_analysis/aggr_stat_line.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.cc b/met/src/tools/core/stat_analysis/aggr_stat_line.cc index 9551cbbf49..24bfcfda92 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -3511,7 +3511,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, if(job.obs_init_hour.n() == n_term) fcst_term.obs_init_hour.set(job.obs_init_hour[i]); if(job.fcst_var.n() == n_term) fcst_term.fcst_var.set(job.fcst_var[i]); if(job.obs_var.n() == n_term) fcst_term.obs_var.set(job.obs_var[i]); - if(job.fcst_lev.n() == n_term) fcst_term.fcst_lev.add(job.fcst_lev[i]); + if(job.fcst_lev.n() == n_term) fcst_term.fcst_lev.set(job.fcst_lev[i]); if(job.obs_lev.n() == n_term) fcst_term.obs_lev.set(job.obs_lev[i]); if(job.obtype.n() == n_term) fcst_term.obtype.set(job.obtype[i]); if(job.vx_mask.n() == n_term) fcst_term.vx_mask.set(job.vx_mask[i]); @@ -3519,6 +3519,7 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, if(job.interp_pnts.n() == n_term) fcst_term.interp_pnts.set(job.interp_pnts[i]); if(job.fcst_thresh.n() == n_term) fcst_term.fcst_thresh.set(job.fcst_thresh[i]); if(job.obs_thresh.n() == n_term) fcst_term.obs_thresh.set(job.obs_thresh[i]); + if(job.line_type.n() == n_term) fcst_term.line_type.set(job.line_type[i]); if(job.column.n() == n_term) fcst_term.column.set(job.column[i]); if(job.weight.n() == n_term) fcst_term.weight.set(job.weight[i]); From cb8f0c00335f21d43ef76bd926e914c83d377a22 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 27 Aug 2021 16:22:41 -0600 Subject: [PATCH 55/60] Per #1788, tweak the ss_index logic slightly to make it more efficient. For each input STATLine, instead of checking all the index terms, stop searching after the first match. This assumes that each input line should be used for only one of the terms... and I think that's a pretty reasonable assumption. --- met/src/tools/core/stat_analysis/skill_score_index_job.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.cc b/met/src/tools/core/stat_analysis/skill_score_index_job.cc index e46f917033..998d97dbc0 100644 --- a/met/src/tools/core/stat_analysis/skill_score_index_job.cc +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.cc @@ -197,7 +197,7 @@ bool SSIndexJobInfo::add(STATLine &line) { } // Check the reference model job - if(ref_job[i].is_keeper(line)) { + if(!keep && ref_job[i].is_keeper(line)) { keep = true; n_ref_lines.inc(i, 1); @@ -212,6 +212,9 @@ bool SSIndexJobInfo::add(STATLine &line) { ref_cts[i].cts += ctc; } } + + // Break out of the loop after the first job match + if(keep) break; } // Track the unique intialization times From 806f0cc4e00ee893e93baeeea7477f284829aa21 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 30 Aug 2021 11:17:01 -0600 Subject: [PATCH 56/60] Per #1788, minor tweaks to the STAT-Analysis docs. --- met/docs/Users_Guide/stat-analysis.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index 933cc9d2ae..f970ee78b7 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -55,7 +55,7 @@ When aggregating the matched pair line type (MPR) and computing an output contin Skill Score Index ~~~~~~~~~~~~~~~~~ -The Stat-Analysis "ss_index", "go_index", and "cbs_index" jobs calculate skill score indices by weighting scores for different meteorological fields at different pressure levels and for different lead times. Pre-defined configuration files are provided for the GO Index and CBS Index which are special cases of the highly configurable skill score index job. +The Stat-Analysis "ss_index", "go_index", and "cbs_index" jobs calculate skill score indices by weighting scores for meteorological fields at different levels and lead times. Pre-defined configuration files are provided for the GO Index and CBS Index which are special cases of the highly configurable skill score index job. In general, a skill score index is computed over several terms and the number and definition of those terms is configurable. It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases. When configuring a skill score index job, the following requirements apply: @@ -94,7 +94,7 @@ When running a skill score index job using the "-out_stat" job command option, a GO Index ~~~~~~~~ -The "go_index" job is a special case of the "ss_index" job, described in :numref:`StA_Skill-Score-Index`. The GO Index is a weighted average of 48 skill scores of RMSE statistics for wind speed, dew point temperature, temperature, height, and pressure at several levels in the atmosphere. The variables, levels, and lead times included in the index are shown in :numref:`compute_GO_Index` and are defined by the default STATAnalysisConfig_GO_Index configuration file. The partial sums (SL1L2 lines in the STAT output) for each of these variables at each level and lead time must have been computed in a previous step. The Stat-Analysis tool then uses the weights in :numref:`compute_GO_Index` to compute values for the GO Index. +The "go_index" job is a special case of the "ss_index" job, described in :numref:`StA_Skill-Score-Index`. The GO Index is a weighted average of 48 skill scores of RMSE statistics for wind speed, dew point temperature, temperature, height, and pressure at several levels in the atmosphere. The variables, levels, and lead times included in the index are listed in :numref:`compute_GO_Index` and are defined by the default "STATAnalysisConfig_GO_Index" configuration file. The partial sums (SL1L2 lines in the STAT output) for each of these variables at each level and lead time must have been computed in a previous step. The Stat-Analysis tool then uses the weights in :numref:`compute_GO_Index` to compute values for the GO Index. .. _compute_GO_Index: @@ -192,7 +192,7 @@ The "go_index" job is a special case of the "ss_index" job, described in :numref CBS Index ~~~~~~~~~ -The "cbs_index" job is a special case of the "ss_index" job, described in :numref:`StA_Skill-Score-Index`. The CBS Index is a weighted average of 40 skill scores of RMSE statistics for mean sea level pressure, height, and wind speed at multiple levels computed over the northern hemisphere, southern hemisphere and the tropics. The variables, levels, lead times, and regions included in the index are shown in :numref:`compute_CBS_Index` and are defined by a default Stat-Analysis configuration file. The partial sums (SL1L2 lines in the STAT output) for each of these variables for each level, lead time, and masking region must have been computed in a previous step. The Stat-Analysis tool then uses the weights in :numref:`compute_CBS_Index` to compute values for the CBS Index. +The "cbs_index" job is a special case of the "ss_index" job, described in :numref:`StA_Skill-Score-Index`. The CBS Index is a weighted average of 40 skill scores of RMSE statistics for mean sea level pressure, height, and wind speed at multiple levels computed over the northern hemisphere, southern hemisphere and the tropics. The variables, levels, lead times, and regions included in the index are listed in :numref:`compute_CBS_Index` and are defined by the default "STATAnalysisConfig_CBS_Index" configuration file. The partial sums (SL1L2 lines in the STAT output) for each of these variables for each level, lead time, and masking region must have been computed in a previous step. The Stat-Analysis tool then uses the weights in :numref:`compute_CBS_Index` to compute values for the CBS Index. .. _compute_CBS_Index: From 9b8d7dd2e616662c8d09c03e11d5d29e2ff48282 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 30 Aug 2021 16:20:53 -0600 Subject: [PATCH 57/60] Per #1788, add ss_index_vld_thresh option to existing STAT-Analysis config files. --- met/data/config/STATAnalysisConfig_CBS_Index | 9 ++++++++- met/data/config/STATAnalysisConfig_GO_Index | 9 ++++++++- met/data/config/STATAnalysisConfig_default | 9 ++++++++- test/config/STATAnalysisConfig_APCP_HIRA | 9 ++++++++- test/config/STATAnalysisConfig_SFC_SS_Index | 9 ++++++++- test/config/STATAnalysisConfig_climo | 9 ++++++++- test/config/STATAnalysisConfig_filter_times | 9 ++++++++- test/config/STATAnalysisConfig_grid_stat | 9 ++++++++- test/config/STATAnalysisConfig_point_stat | 9 ++++++++- test/config/STATAnalysisConfig_ramps | 9 ++++++++- 10 files changed, 80 insertions(+), 10 deletions(-) diff --git a/met/data/config/STATAnalysisConfig_CBS_Index b/met/data/config/STATAnalysisConfig_CBS_Index index dbb5b9cf65..989b1de4ea 100644 --- a/met/data/config/STATAnalysisConfig_CBS_Index +++ b/met/data/config/STATAnalysisConfig_CBS_Index @@ -121,7 +121,14 @@ boot = { //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "CBS_INDEX"; +// +// Skill score index options +// +ss_index_name = "CBS_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; diff --git a/met/data/config/STATAnalysisConfig_GO_Index b/met/data/config/STATAnalysisConfig_GO_Index index 8ce63fd944..a82daa1710 100644 --- a/met/data/config/STATAnalysisConfig_GO_Index +++ b/met/data/config/STATAnalysisConfig_GO_Index @@ -129,7 +129,14 @@ boot = { //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "GO_INDEX"; +// +// Skill score index options +// +ss_index_name = "GO_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; diff --git a/met/data/config/STATAnalysisConfig_default b/met/data/config/STATAnalysisConfig_default index e1ec9912a0..193faf7d3b 100644 --- a/met/data/config/STATAnalysisConfig_default +++ b/met/data/config/STATAnalysisConfig_default @@ -108,7 +108,14 @@ wmo_fisher_stats = [ "CNT:PR_CORR", "CNT:SP_CORR", //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "SS_INDEX"; +// +// Skill score index options +// +ss_index_name = "SS_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; diff --git a/test/config/STATAnalysisConfig_APCP_HIRA b/test/config/STATAnalysisConfig_APCP_HIRA index d7a666674f..e76c4a8ffc 100644 --- a/test/config/STATAnalysisConfig_APCP_HIRA +++ b/test/config/STATAnalysisConfig_APCP_HIRA @@ -98,7 +98,14 @@ boot = { //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "SS_INDEX"; +// +// Skill score index options +// +ss_index_name = "SS_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; diff --git a/test/config/STATAnalysisConfig_SFC_SS_Index b/test/config/STATAnalysisConfig_SFC_SS_Index index 1311321c91..ca08d2565c 100644 --- a/test/config/STATAnalysisConfig_SFC_SS_Index +++ b/test/config/STATAnalysisConfig_SFC_SS_Index @@ -95,7 +95,14 @@ boot = { //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "SFC_SS_INDEX"; +// +// Skill score index options +// +ss_index_name = "SS_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; diff --git a/test/config/STATAnalysisConfig_climo b/test/config/STATAnalysisConfig_climo index bd604acf3e..608bd3b255 100644 --- a/test/config/STATAnalysisConfig_climo +++ b/test/config/STATAnalysisConfig_climo @@ -104,7 +104,14 @@ wmo_fisher_stats = [ "CNT:PR_CORR", "CNT:SP_CORR", //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "SS_INDEX"; +// +// Skill score index options +// +ss_index_name = "SS_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; diff --git a/test/config/STATAnalysisConfig_filter_times b/test/config/STATAnalysisConfig_filter_times index ced2004239..6af000732f 100644 --- a/test/config/STATAnalysisConfig_filter_times +++ b/test/config/STATAnalysisConfig_filter_times @@ -101,7 +101,14 @@ boot = { //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "SS_INDEX"; +// +// Skill score index options +// +ss_index_name = "SS_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; diff --git a/test/config/STATAnalysisConfig_grid_stat b/test/config/STATAnalysisConfig_grid_stat index b8e9fbb9ba..3365bf5f24 100644 --- a/test/config/STATAnalysisConfig_grid_stat +++ b/test/config/STATAnalysisConfig_grid_stat @@ -94,7 +94,14 @@ boot = { //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "SS_INDEX"; +// +// Skill score index options +// +ss_index_name = "SS_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; diff --git a/test/config/STATAnalysisConfig_point_stat b/test/config/STATAnalysisConfig_point_stat index c6dd05832f..82a68fbaf6 100644 --- a/test/config/STATAnalysisConfig_point_stat +++ b/test/config/STATAnalysisConfig_point_stat @@ -123,7 +123,14 @@ boot = { //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "SS_INDEX"; +// +// Skill score index options +// +ss_index_name = "SS_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; diff --git a/test/config/STATAnalysisConfig_ramps b/test/config/STATAnalysisConfig_ramps index cd7bec39a3..aca6de0045 100644 --- a/test/config/STATAnalysisConfig_ramps +++ b/test/config/STATAnalysisConfig_ramps @@ -93,7 +93,14 @@ boot = { //////////////////////////////////////////////////////////////////////////////// -ss_index_name = "SS_INDEX"; +// +// Skill score index options +// +ss_index_name = "SS_INDEX"; +ss_index_vld_thresh = 1.0; + +//////////////////////////////////////////////////////////////////////////////// + hss_ec_value = NA; rank_corr_flag = TRUE; vif_flag = FALSE; From 736c489741dd7ca56e9e43c65025487202bb137f Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 30 Aug 2021 16:22:55 -0600 Subject: [PATCH 58/60] Per #1788, add StatAnalysisJob ss_index_vld_thresh entry and parse it from the STAT-Analysis config file. --- met/src/basic/vx_config/config_constants.h | 69 ++++++++++--------- met/src/libcode/vx_analysis_util/stat_job.cc | 14 ++++ met/src/libcode/vx_analysis_util/stat_job.h | 5 ++ .../core/stat_analysis/stat_analysis_job.cc | 13 +++- 4 files changed, 66 insertions(+), 35 deletions(-) diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index 3b78c305f8..5a29f44347 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -758,40 +758,41 @@ static const char conf_val_beta[] = "BETA"; // STAT-Analysis specific parameter key names // -static const char conf_key_fcst_lead[] = "fcst_lead"; -static const char conf_key_obs_lead[] = "obs_lead"; -static const char conf_key_fcst_valid_beg[] = "fcst_valid_beg"; -static const char conf_key_fcst_valid_end[] = "fcst_valid_end"; -static const char conf_key_fcst_valid_hour[] = "fcst_valid_hour"; -static const char conf_key_obs_valid_beg[] = "obs_valid_beg"; -static const char conf_key_obs_valid_end[] = "obs_valid_end"; -static const char conf_key_obs_valid_hour[] = "obs_valid_hour"; -static const char conf_key_fcst_init_beg[] = "fcst_init_beg"; -static const char conf_key_fcst_init_end[] = "fcst_init_end"; -static const char conf_key_fcst_init_hour[] = "fcst_init_hour"; -static const char conf_key_obs_init_beg[] = "obs_init_beg"; -static const char conf_key_obs_init_end[] = "obs_init_end"; -static const char conf_key_obs_init_hour[] = "obs_init_hour"; -static const char conf_key_fcst_var[] = "fcst_var"; -static const char conf_key_obs_var[] = "obs_var"; -static const char conf_key_fcst_units[] = "fcst_units"; -static const char conf_key_obs_units[] = "obs_units"; -static const char conf_key_fcst_lev[] = "fcst_lev"; -static const char conf_key_obs_lev[] = "obs_lev"; -static const char conf_key_vx_mask[] = "vx_mask"; -static const char conf_key_interp_mthd[] = "interp_mthd"; -static const char conf_key_interp_pnts[] = "interp_pnts"; -static const char conf_key_fcst_thresh[] = "fcst_thresh"; -static const char conf_key_obs_thresh[] = "obs_thresh"; -static const char conf_key_alpha[] = "alpha"; -static const char conf_key_line_type[] = "line_type"; -static const char conf_key_column[] = "column"; -static const char conf_key_out_alpha[] = "out_alpha"; -static const char conf_key_vif_flag[] = "vif_flag"; -static const char conf_key_wmo_sqrt_stats[] = "wmo_sqrt_stats"; -static const char conf_key_wmo_fisher_stats[] = "wmo_fisher_stats"; -static const char conf_key_jobs[] = "jobs"; -static const char conf_key_ss_index_name[] = "ss_index_name"; +static const char conf_key_fcst_lead[] = "fcst_lead"; +static const char conf_key_obs_lead[] = "obs_lead"; +static const char conf_key_fcst_valid_beg[] = "fcst_valid_beg"; +static const char conf_key_fcst_valid_end[] = "fcst_valid_end"; +static const char conf_key_fcst_valid_hour[] = "fcst_valid_hour"; +static const char conf_key_obs_valid_beg[] = "obs_valid_beg"; +static const char conf_key_obs_valid_end[] = "obs_valid_end"; +static const char conf_key_obs_valid_hour[] = "obs_valid_hour"; +static const char conf_key_fcst_init_beg[] = "fcst_init_beg"; +static const char conf_key_fcst_init_end[] = "fcst_init_end"; +static const char conf_key_fcst_init_hour[] = "fcst_init_hour"; +static const char conf_key_obs_init_beg[] = "obs_init_beg"; +static const char conf_key_obs_init_end[] = "obs_init_end"; +static const char conf_key_obs_init_hour[] = "obs_init_hour"; +static const char conf_key_fcst_var[] = "fcst_var"; +static const char conf_key_obs_var[] = "obs_var"; +static const char conf_key_fcst_units[] = "fcst_units"; +static const char conf_key_obs_units[] = "obs_units"; +static const char conf_key_fcst_lev[] = "fcst_lev"; +static const char conf_key_obs_lev[] = "obs_lev"; +static const char conf_key_vx_mask[] = "vx_mask"; +static const char conf_key_interp_mthd[] = "interp_mthd"; +static const char conf_key_interp_pnts[] = "interp_pnts"; +static const char conf_key_fcst_thresh[] = "fcst_thresh"; +static const char conf_key_obs_thresh[] = "obs_thresh"; +static const char conf_key_alpha[] = "alpha"; +static const char conf_key_line_type[] = "line_type"; +static const char conf_key_column[] = "column"; +static const char conf_key_out_alpha[] = "out_alpha"; +static const char conf_key_vif_flag[] = "vif_flag"; +static const char conf_key_wmo_sqrt_stats[] = "wmo_sqrt_stats"; +static const char conf_key_wmo_fisher_stats[] = "wmo_fisher_stats"; +static const char conf_key_jobs[] = "jobs"; +static const char conf_key_ss_index_name[] = "ss_index_name"; +static const char conf_key_ss_index_vld_thresh[] = "ss_index_vld_thresh"; // // MODE specific parameter key names diff --git a/met/src/libcode/vx_analysis_util/stat_job.cc b/met/src/libcode/vx_analysis_util/stat_job.cc index 90b2d97500..70464037c6 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.cc +++ b/met/src/libcode/vx_analysis_util/stat_job.cc @@ -204,6 +204,8 @@ void STATAnalysisJob::clear() { n_boot_rep = bad_data_int; ss_index_name.clear(); + ss_index_vld_thresh = bad_data_double; + hss_ec_value = bad_data_double; rank_corr_flag = false; vif_flag = false; @@ -348,6 +350,8 @@ void STATAnalysisJob::assign(const STATAnalysisJob & aj) { n_boot_rep = aj.n_boot_rep; ss_index_name = aj.ss_index_name; + ss_index_vld_thresh = aj.ss_index_vld_thresh; + hss_ec_value = aj.hss_ec_value; rank_corr_flag = aj.rank_corr_flag; vif_flag = aj.vif_flag; @@ -634,6 +638,9 @@ void STATAnalysisJob::dump(ostream & out, int depth) const { out << prefix << "ss_index_name = " << ss_index_name << "\n"; + out << prefix << "ss_index_vld_thresh = " + << ss_index_vld_thresh << "\n"; + out << prefix << "hss_ec_value = " << hss_ec_value << "\n"; @@ -1616,6 +1623,10 @@ void STATAnalysisJob::parse_job_command(const char *jobstring) { ss_index_name = jc_array[i+1].c_str(); i++; } + else if(jc_array[i] == "-ss_index_vld_thresh") { + ss_index_vld_thresh = atof(jc_array[i+1].c_str()); + i++; + } else if(jc_array[i] == "-hss_ec_value") { hss_ec_value = atof(jc_array[i+1].c_str()); i++; @@ -2793,6 +2804,9 @@ ConcatString STATAnalysisJob::get_jobstring() const { // ss_index_name js << "-ss_index_name " << ss_index_name << " "; + + // ss_index_vld_thresh + js << "-ss_index_vld_thresh " << ss_index_vld_thresh << " "; } // Jobs which write MCTC or MCTS output diff --git a/met/src/libcode/vx_analysis_util/stat_job.h b/met/src/libcode/vx_analysis_util/stat_job.h index bef4f68f1e..bed32cdc1a 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.h +++ b/met/src/libcode/vx_analysis_util/stat_job.h @@ -329,6 +329,11 @@ class STATAnalysisJob { // ConcatString ss_index_name; + // + // Required ratio of valid skill score index terms + // + double ss_index_vld_thresh; + // // MCTS HSS Expected Correct rate // diff --git a/met/src/tools/core/stat_analysis/stat_analysis_job.cc b/met/src/tools/core/stat_analysis/stat_analysis_job.cc index 7854ca8f3c..85df53e471 100644 --- a/met/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/met/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -121,7 +121,18 @@ void set_job_from_config(MetConfig &c, STATAnalysisJob &job) { job.set_boot_rng(boot_info.rng.c_str()); job.set_boot_seed(boot_info.seed.c_str()); - job.ss_index_name = c.lookup_string(conf_key_ss_index_name); + job.ss_index_name = c.lookup_string(conf_key_ss_index_name); + job.ss_index_vld_thresh = c.lookup_double(conf_key_ss_index_vld_thresh); + + // Check that ss_index_vld_thresh is in [0, 1]. + if(job.ss_index_vld_thresh < 0.0 || + job.ss_index_vld_thresh > 1.0) { + mlog << Error << "\nset_job_from_config() -> " + << "\"" << conf_key_ss_index_vld_thresh << "\" (" + << job.ss_index_vld_thresh << ") must be set between 0 and 1.\n\n"; + throw(1); + } + job.hss_ec_value = c.lookup_double(conf_key_hss_ec_value); job.rank_corr_flag = (int) c.lookup_bool(conf_key_rank_corr_flag); job.vif_flag = (int) c.lookup_bool(conf_key_vif_flag); From b532b02654830436542ba456fbaaec1b58805424 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 30 Aug 2021 16:25:31 -0600 Subject: [PATCH 59/60] Per #1788, update stat_analysis code to check ss_index_vld_thresh setting when computing the skill score index. Also update the Stat-Analysis chapter of the User's Guide. --- met/docs/Users_Guide/stat-analysis.rst | 16 ++++++++++++---- .../tools/core/stat_analysis/aggr_stat_line.cc | 5 +++-- .../core/stat_analysis/skill_score_index_job.cc | 12 ++++++++++++ .../core/stat_analysis/skill_score_index_job.h | 3 +++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index f970ee78b7..80c843b9a3 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -512,11 +512,19 @@ ___________________ .. code-block:: none - column = []; - weight = []; - ss_index_name = "SS_INDEX"; + column = []; + weight = []; -The column, weight, and ss_index_name entries are used to define a skill score index. The column and weight arrays can either be set to a constant value of length one or specify a separate value for each term of the index. The ss_index_name entry is a string which defines the output name for the current skill score index configuration. +The column and weight entries are used to define a skill score index. They can either be set to a constant value of length one or specify a separate value for each term of the index. + +___________________ + +.. code-block:: none + + ss_index_name = "SS_INDEX"; + ss_index_vld_thresh = 1.0; + +The ss_index_name and ss_index_vld_thresh options are used to define a skill score index. The ss_index_name entry is a string which defines the output name for the current skill score index configuration. The ss_index_vld_thresh entry is a number between 0.0 and 1.0 that defines the required ratio of valid terms. If the ratio of valid skill score index terms to the total is less than than this number, no output is written for that case. The default value of 1.0 indicates that all terms are required. ___________________ diff --git a/met/src/tools/core/stat_analysis/aggr_stat_line.cc b/met/src/tools/core/stat_analysis/aggr_stat_line.cc index 24bfcfda92..3e66de254e 100644 --- a/met/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/met/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -3405,9 +3405,10 @@ void aggr_ss_index(LineDataFile &f, STATAnalysisJob &job, int i, n_term; // - // Store the index name + // Store the index name and valid data threshold // - cur.job_info.ss_index_name = job.ss_index_name; + cur.job_info.ss_index_name = job.ss_index_name; + cur.job_info.ss_index_vld_thresh = job.ss_index_vld_thresh; // // Check that the -model option has been supplied exactly 2 times. diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.cc b/met/src/tools/core/stat_analysis/skill_score_index_job.cc index 998d97dbc0..7cfac5a250 100644 --- a/met/src/tools/core/stat_analysis/skill_score_index_job.cc +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.cc @@ -83,6 +83,7 @@ void SSIndexJobInfo::clear() { // Initialize ss_index_name.clear(); + ss_index_vld_thresh = 0.0; fcst_model.clear(); ref_model.clear(); n_term = 0; @@ -104,6 +105,7 @@ void SSIndexJobInfo::clear() { void SSIndexJobInfo::assign(const SSIndexJobInfo &c) { ss_index_name = c.ss_index_name; + ss_index_vld_thresh = c.ss_index_vld_thresh; fcst_model = c.fcst_model; ref_model = c.ref_model; n_term = c.n_term; @@ -319,6 +321,16 @@ SSIDXData SSIndexJobInfo::compute_ss_index() { } // end for i + // Check the required ratio of valid terms + if(((double) n_vld / n_term) < ss_index_vld_thresh) { + mlog << Warning << "\nSSIndexJobInfo::compute_ss_index() -> " + << "skipping " << ss_index_name + << " since the ratio of valid terms " << n_vld << "/" + << n_term << " < " << ss_index_vld_thresh << "!\n\n"; + data.n_vld = 0; + return(data); + } + // Compute the weighted average of the skill scores if(is_eq(weight_sum, 0.0)) ss_avg = bad_data_double; else ss_avg = ss_sum/weight_sum; diff --git a/met/src/tools/core/stat_analysis/skill_score_index_job.h b/met/src/tools/core/stat_analysis/skill_score_index_job.h index 9e14341906..60d2350171 100644 --- a/met/src/tools/core/stat_analysis/skill_score_index_job.h +++ b/met/src/tools/core/stat_analysis/skill_score_index_job.h @@ -62,6 +62,9 @@ class SSIndexJobInfo { // Index name ConcatString ss_index_name; + // Valid data threshold + double ss_index_vld_thresh; + // Forecast and reference model names ConcatString fcst_model, ref_model; From bc5491c36fcbca9583f50efe23721d92c7df3f15 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 30 Aug 2021 17:07:23 -0600 Subject: [PATCH 60/60] Per #1788, tweak one test to exercise a non-default ss_index_vld_thresh value. --- test/xml/unit_ref_config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/xml/unit_ref_config.xml b/test/xml/unit_ref_config.xml index a18fffa811..648da522a9 100644 --- a/test/xml/unit_ref_config.xml +++ b/test/xml/unit_ref_config.xml @@ -964,7 +964,7 @@ -lookin &OUTPUT_DIR;/ref_config/point_stat/AFWAv3.4_Noahv2.7.1/ \ -job go_index \ -by FCST_INIT_BEG,VX_MASK,OBTYPE -set_hdr DESC Noahv3.3_vs_v2.7.1 \ - -model AFWAv3.4_Noahv3.3,AFWAv3.4_Noahv2.7.1 \ + -model AFWAv3.4_Noahv3.3,AFWAv3.4_Noahv2.7.1 -ss_index_vld_thresh 0.5 \ -out_stat &OUTPUT_DIR;/ref_config/stat_analysis/go_index_by_option.stat