diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index a768ac23e6..85a71cc97f 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -138,44 +138,65 @@ watch_warn = { } // -// Diagnostics to be extracted -// -diag_name = []; +// Metadata for diagnostic sources +// +diag_info_map = [ + { + diag_source = "CIRA_DIAG_RT"; + track_source = "GFS"; + field_source = "GFS_0p50"; + match_to_track = [ "GFS" ]; + diag_name = []; + }, + { + diag_source = "SHIPS_DIAG_RT"; + track_source = "SHIPS_TRK"; + field_source = "GFS_0p50"; + match_to_track = [ "OFCL" ]; + diag_name = []; + } +]; // -// Unit conversions to be applied based on diagnostic names and units +// Unit conversions for diagnostic sources, names, and units // diag_convert_map = [ - { source = "TCDIAG"; - key = [ "(10C)", "(10KT)", "(10M/S)" ]; - convert(x) = x / 10; }, - - { source = "LSDIAG_RT"; - key = [ "LAT", "LON", "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", - "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", - "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", - "V500", "V300", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", - "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", - "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", - "PW20", "PW21" ]; - convert(x) = x / 10; }, - - { source = "LSDIAG_RT"; - key = [ "VVAV", "VMFX", "VVAC" ]; - convert(x) = x / 100; }, - - { source = "LSDIAG_RT"; - key = [ "TADV" ]; - convert(x) = x / 1000000; }, - - { source = "LSDIAG_RT"; - key = [ "Z850", "D200", "TGRD", "DIVC" ]; - convert(x) = x / 10000000; }, - - { source = "LSDIAG_RT"; - key = [ "PENC", "PENV" ]; - convert(x) = x / 10 + 1000; } - + { + diag_source = "CIRA_DIAG"; + key = [ "(10C)", "(10KT)", "(10M/S)" ]; + convert(x) = x / 10; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "LAT", "LON", "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", + "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", + "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", + "V500", "V300", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", + "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", + "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", + "PW20", "PW21" ]; + convert(x) = x / 10; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "VVAV", "VMFX", "VVAC" ]; + convert(x) = x / 100; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "TADV" ]; + convert(x) = x / 1000000; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "Z850", "D200", "TGRD", "DIVC" ]; + convert(x) = x / 10000000; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "PENC", "PENV" ]; + convert(x) = x / 10 + 1000; + } ]; // diff --git a/data/table_files/met_header_columns_V11.0.txt b/data/table_files/met_header_columns_V11.0.txt index 44f5626b09..c393863057 100644 --- a/data/table_files/met_header_columns_V11.0.txt +++ b/data/table_files/met_header_columns_V11.0.txt @@ -37,5 +37,5 @@ V11.0 : MODE : OBJ : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VAL V11.0 : MODE : CTS : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS V11.0 : TCST : TCMPR : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX LEVEL WATCH_WARN INITIALS ALAT ALON BLAT BLON TK_ERR X_ERR Y_ERR ALTK_ERR CRTK_ERR ADLAND BDLAND AMSLP BMSLP AMAX_WIND BMAX_WIND AAL_WIND_34 BAL_WIND_34 ANE_WIND_34 BNE_WIND_34 ASE_WIND_34 BSE_WIND_34 ASW_WIND_34 BSW_WIND_34 ANW_WIND_34 BNW_WIND_34 AAL_WIND_50 BAL_WIND_50 ANE_WIND_50 BNE_WIND_50 ASE_WIND_50 BSE_WIND_50 ASW_WIND_50 BSW_WIND_50 ANW_WIND_50 BNW_WIND_50 AAL_WIND_64 BAL_WIND_64 ANE_WIND_64 BNE_WIND_64 ASE_WIND_64 BSE_WIND_64 ASW_WIND_64 BSW_WIND_64 ANW_WIND_64 BNW_WIND_64 ARADP BRADP ARRP BRRP AMRD BMRD AGUSTS BGUSTS AEYE BEYE ADIR BDIR ASPEED BSPEED ADEPTH BDEPTH NUM_MEMBERS TRACK_SPREAD TRACK_STDEV MSLP_STDEV MAX_WIND_STDEV -V11.0 : TCST : TCDIAG : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX DIAG_SOURCE (N_DIAG) DIAG_[0-9]* VALUE_[0-9]* +V11.0 : TCST : TCDIAG : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX DIAG_SOURCE TRACK_SOURCE FIELD_SOURCE (N_DIAG) DIAG_[0-9]* VALUE_[0-9]* V11.0 : TCST : PROBRIRW : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE ALAT ALON BLAT BLON INITIALS TK_ERR X_ERR Y_ERR ADLAND BDLAND RIRW_BEG RIRW_END RIRW_WINDOW AWIND_END BWIND_BEG BWIND_END BDELTA BDELTA_MAX BLEVEL_BEG BLEVEL_END (N_THRESH) THRESH_[0-9]* PROB_[0-9]* diff --git a/docs/Users_Guide/tc-pairs.rst b/docs/Users_Guide/tc-pairs.rst index 18c1006854..40162dee4d 100644 --- a/docs/Users_Guide/tc-pairs.rst +++ b/docs/Users_Guide/tc-pairs.rst @@ -14,10 +14,39 @@ Scientific and statistical aspects .. _TC-Pairs_Diagnostics: -Storm Diagnostics +TC Diagnostics ----------------- -TODO: Add a paragraph about storm diagnostics, describing what they are, why they are important, and how they can be generated. +TC diagnostics provide information about a TC's structure or its environment. Each TC diagnostic is a single-valued measure that corresponds to some aspect of the storm itself or the surrounding large-scale environment. TC diagnostics can be derived from observational analyses, model fields, or even satellite observations. Examples include: + + * Inner core diagnostics provide information about the structure of the storm near the storm center. Examples include the intensity of the storm and the radius of maximum winds. + + * Large scale diagnostics provide information about quantities that characterize its environment. Examples include environmental vertical wind shear, total precipitable water, the average relative humidity, measures of convective instability, and the upper bound of intensity that a storm may be expected to achieve in its current environment. These diagnostics are typically derived from model fields as an average of the quantity of interest over either a circular area or an annulus centered on the storm center. Often, the storm center is taken to be the underlying model's storm center. In other cases, the diagnostics may be computed along some other specified track. + + * Ocean-based diagnostics provide information about the sea's thermal characteristics in the vicinity of the storm center. Examples include the sea surface temperature, ocean heat content, and the depth of warm water of a given temperature. + + * Satellite-based diagnostics provide information about the storm structure as observed by geostationary satellite infrared imagery. Examples include information about the shape and extent of the cold-cirrus canopy of the TC and whether patterns are present that may portend intensification. + +Diagnostics are critically important for training and running statistical-dynamical models that predict a TC's intensity or size. One of the most well-known diagnostics sets is that of the Statistical Hurricane Intensity Prediction Scheme (SHIPS), which supports predictions of TC intensity. A large 30-year development dataset of TC diagnostics has been retrospectively derived to support the training of the SHIPS intensity model as well as other related models such as the Logistic Growth Equation Model (LGEM), SHIPS Rapid Intensification Index (SHIPS-RII), and others. These diagnostics, called *lsdiag* for "large scale" environment, are computed using a *perfect prog* approach in which the diagnostics are computed on the reference model's verifying analyses to generate a set of time-dependent diagnostics from t=0 out to the desired maximum forecast lead time. This is repeated for each initialization, building up a full history of diagnostics for each storm. By using the subsequent verifying analysis for later lead times, the model is taken to be "perfect", removing the impact of model forecast errors. The resulting developmental dataset is ideal for training statistical-dynamical models such as SHIPS. To generate forecasts in real-time, the diagnostics are computed along a forecast track (often taken to be the National Hurricane Center's official forecast) using the fields of the underlying NWP model (e.g, the Global Forecast System, or GFS model). The resulting diagnostics are then used as *predictors* in models like SHIPS and LGEM to predict a TC's future intensity or probability of undergoing rapid intensification. + +Beside their use in TC prediction, TC diagnostics can be very useful to forecasters to understand the forecast scenario. They are also useful to model developers for evaluation of model errors and understanding model performance under different environmental conditions. For instance, a modeler may wish to understand their model's track biases under conditions of high vertical wind shear. TC diagnostics can also be used to understand the sensitivity of the model's intensity predictions to oceanic conditions such as upwelling. The TC-Pairs tool allows filtering and subsetting based on the values of one or several TC diagnostics. + +As of MET v11.0.0, two types of TC diagnostics are supported in TC-Pairs: + + .. SHIPS_DIAG_DEV: Includes a plethora of inner core, environmental, oceanic, and satellite-based diagnostics. These diagnostics are computed using the *perfect prog* approach. + + * SHIPS_DIAG_RT: Real-time SHIPS diagnostics computed from a NWP model such as the Global Forecast System (GFS) model along a GFS forecast track defined by a SHIPS-specific tracker. Note that these SHIPS-derived forecast tracks do not appear in the NHC adeck data. + + * CIRA_DIAG_RT: Real-time model-based diagnostics computed along the model's predicted track. + +Diagnostics from the SHIPS Development Datasets (SHIPS_DIAG_DEV) will be supported in a future release of MET. + +A future version of MET will also allow the CIRA model diagnostics to be computed directly from model forecast fields. Until then, users may obtain the SHIPS diagnostics at the following locations: + + * SHIPS_DIAG_DEV: https://rammb2.cira.colostate.edu/research/tropical-cyclones/ships/#DevelopmentalData + + * SHIPS_DIAG_RT: https://ftp.nhc.noaa.gov/atcf/lsdiag/ + .. _TC-Pairs_Practical-information: @@ -58,7 +87,7 @@ Required arguments for tc_pairs Optional arguments for tc_pairs ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -5. The **-diag source path** argument indicates the TC-Pairs acceptable format data containing the tropical cyclone diagnostics dataset corresponding to the adeck tracks. The **source** can be set to TCDIAG, LSDIAG_RT, or LSDIAG_DEV to indicate the input diagnostics data source. The **path** argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. +5. The **-diag source path** argument indicates the TC-Pairs acceptable format data containing the tropical cyclone diagnostics dataset corresponding to the adeck tracks. The **source** can be set to CIRA_DIAG_RT or SHIPS_DIAG_RT to indicate the input diagnostics data source. The **path** argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. Support for additional diagnostic sources will be added in future releases. 6. The -**out base** argument indicates the path of the output file base. This argument overrides the default output file base (**./out_tcmpr**). @@ -70,8 +99,6 @@ This tool currently only supports the rapid intensification (**RI**) edeck proba At least one **-adeck** or **-edeck** option must be specified. The **-adeck, -edeck**, and **-bdeck** options may optionally be followed with **suffix=string** to append that string to all model names found within that data source. This option may be useful when processing track data from two different sources which reuse the same model names. -The **-diag** option may optionally be followed with **model=string** to override the model name of the tracks to which those diagnostics correspond. The **string** specifies a comma-separated list of one or more ATCF ID's to which these diagnostics should be paired (e.g. **model=OFCL,SHIP**). - An example of the tc_pairs calling sequence is shown below: .. code-block:: none @@ -258,52 +285,75 @@ ____________________ .. code-block:: none - diag_name = []; - -The **diag_name** entry specifies a comma-separated list of strings for the tropical cyclone diagnostics of interest. This applies when the **-tcdiag** and/or **-lsdiag** command line options have been used to provide storm diagnostics data. If a non-zero list of diagnostic names is specified, only those diagnostics appearing in the list are written to the TCDIAG output line type. If defined as an empty list (default), all diagnostics found in the input are written to the TCDIAG output lines. - -A TCMPR line is written to the output for each track point. If diagnostics data is also defined for that track point, a TCDIAG line is written immediately after the corresponding TCMPR line. The contents of that TCDIAG line is determined by diagnostic names requested in the **diag_name** entry. + diag_info_map = [ + { + diag_source = "CIRA_DIAG_RT"; + track_source = "GFS"; + field_source = "GFS_0p50"; + match_to_track = [ "GFS" ]; + diag_name = []; + }, + { + diag_source = "SHIPS_DIAG_RT"; + track_source = "SHIPS_TRK"; + field_source = "GFS_0p50"; + match_to_track = [ "OFCL" ]; + diag_name = []; + } + ]; + +A TCMPR line is written to the output for each track point. If diagnostics data is also defined for that track point, a TCDIAG line is written immediately after the corresponding TCMPR line. The contents of that TCDIAG line is determined by the **diag_info_map** entry. + +The **diag_info_map** entries define how the diagnostics read with the **-diag** command line option should be used. Each array element is a dictionary consisting of entries for **diag_source**, **track_source**, **field_source**, **match_to_track**, and **diag_name**. + +The **diag_source** entry is one of the supported diagnostics data sources. The **track_source** entry is a string defining the ATCF ID of the track data used to define the locations at which diagnostics are computed. This string is written to the **TRACK_SOURCE** column of the TCDIAG output line. The **field_source** entry is a string describing the gridded model data from which the diagnostics are computed. This string is written to the **FIELD_SOURCE** column of the TCDIAG output line type. The **match_to_track** entry specifies a comma-separated list of strings defining the ATCF ID(s) of the tracks to which these diagnostic values should be matched. The **diag_name** entry specifies a comma-separated list of strings for the tropical cyclone diagnostics of interest. If a non-zero list of diagnostic names is specified, only those diagnostics appearing in the list are written to the TCDIAG output line type. If defined as an empty list (default), all diagnostics found in the input are written to the TCDIAG output lines. ____________________ .. code-block:: none - diag_convert_map = [ - { source = "TCDIAG"; - key = [ "(10C)", "(10KT)", "(10M/S)" ]; - convert(x) = x / 10; }, - - { source = "LSDIAG_RT"; - key = [ "LAT", "LON", "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", - "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", - "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", - "V500", "V300", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", - "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", - "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", - "PW20", "PW21" ]; - convert(x) = x / 10; }, - - { source = "LSDIAG_RT"; - key = [ "VVAV", "VMFX", "VVAC" ]; - convert(x) = x / 100; }, - - { source = "LSDIAG_RT"; - key = [ "TADV" ]; - convert(x) = x / 1000000; }, - - { source = "LSDIAG_RT"; - key = [ "Z850", "D200", "TGRD", "DIVC" ]; - convert(x) = x / 10000000; }, - - { source = "LSDIAG_RT"; - key = [ "PENC", "PENV" ]; - convert(x) = x / 10 + 1000; } - - ]; - -The **diag_convert_map** entries define conversion functions to be applied to diagnostics data read with the **-diag** command line option. Each array element is a dictionary consisting of a **source**, **key**, and **convert(x)** entry. - -The **source** is one of the supported diagnostics data sources. The **key** is an array of strings. The strings can specify diagnostic names or units, although units are only checked for **TCDIAG** sources. If both the name and units are specified, the conversion function for the name takes precedence. **convert(x)** is a function of one variable which defines how the diagnostic data should be converted. The defined function is applied to any diagnostic value whose name or units appears in the **key**. + diag_convert_map = [ + { + diag_source = "CIRA_DIAG"; + key = [ "(10C)", "(10KT)", "(10M/S)" ]; + convert(x) = x / 10; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "LAT", "LON", "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", + "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", + "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", + "V500", "V300", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", + "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", + "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", + "PW20", "PW21" ]; + convert(x) = x / 10; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "VVAV", "VMFX", "VVAC" ]; + convert(x) = x / 100; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "TADV" ]; + convert(x) = x / 1000000; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "Z850", "D200", "TGRD", "DIVC" ]; + convert(x) = x / 10000000; + }, + { + diag_source = "SHIPS_DIAG"; + key = [ "PENC", "PENV" ]; + convert(x) = x / 10 + 1000; + } + ]; + +The **diag_convert_map** entries define conversion functions to be applied to diagnostics data read with the **-diag** command line option. Each array element is a dictionary consisting of a **diag_source**, **key**, and **convert(x)** entry. + +The **diag_source** entry is one of the supported diagnostics data sources. Partial string matching logic is applied, so **SHIPS_DIAG** entries are matched to both **SHIPS_DIAG_RT** and **SHIPS_DIAG_DEV** diagnostic sources. The **key** entry is an array of strings. The strings can specify diagnostic names or units, although units are only checked for **CIRA_DIAG** sources. If both the name and units are specified, the conversion function for the name takes precedence. The **convert(x)** entry is a function of one variable which defines how the diagnostic data should be converted. The defined function is applied to any diagnostic value whose name or units appears in the **key**. ____________________ @@ -583,14 +633,20 @@ TC-Pairs produces output in TCST format. The default output file name can be ove - Index of the current track pair * - 16 - DIAG_SOURCE - - Diagnostics data source + - Diagnostics data source indicated by the `-diag` command line option * - 17 + - TRACK_SOURCE + - ATCF ID of the track data used to define the diagnostics + * - 18 + - FIELD_SOURCE + - Description of gridded field data source used to define the diagnostics + * - 19 - N_DIAG - Number of storm diagnostic name and value columns to follow - * - 18 + * - 20 - DIAG_i - Name of the of the ith storm diagnostic (repeated) - * - 19 + * - 21 - VALUE_i - Value of the ith storm diagnostic (repeated) diff --git a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS index a2ae20816b..b3018adf16 100644 --- a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS +++ b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS @@ -131,7 +131,22 @@ watch_warn = { // // Diagnostics to be extracted // -diag_name = [ ${DIAG_NAME} ]; +diag_info_map = [ + { + diag_source = "CIRA_DIAG_RT"; + track_source = "GFS"; + field_source = "GFS_0p50"; + match_to_track = [ "GFS" ]; + diag_name = [ ${CIRA_RT_DIAG_NAME} ]; + }, + { + diag_source = "SHIPS_DIAG_RT"; + track_source = "OFCL"; + field_source = "GFS_0p50"; + match_to_track = [ "OFCL", "SHIP" ]; + diag_name = [ ${SHIPS_RT_DIAG_NAME} ]; + } +]; // // Unit conversions to be applied based on diagnostic names and units diff --git a/internal/test_unit/hdr/met_11_0.hdr b/internal/test_unit/hdr/met_11_0.hdr index b41b168740..4c496af7f9 100644 --- a/internal/test_unit/hdr/met_11_0.hdr +++ b/internal/test_unit/hdr/met_11_0.hdr @@ -34,5 +34,5 @@ MODE_SOA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L MODE_POA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES OBJECT_ID OBJECT_CAT CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST MODE_CTS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS TCST_TCMPR : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE AMODEL BMODEL STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK TOTAL INDEX LEVEL WATCH_WARN INITIALS ALAT ALON BLAT BLON TK_ERR X_ERR Y_ERR ALTK_ERR CRTK_ERR ADLAND BDLAND AMSLP BMSLP AMAX_WIND BMAX_WIND AAL_WIND_34 BAL_WIND_34 ANE_WIND_34 BNE_WIND_34 ASE_WIND_34 BSE_WIND_34 ASW_WIND_34 BSW_WIND_34 ANW_WIND_34 BNW_WIND_34 AAL_WIND_50 BAL_WIND_50 ANE_WIND_50 BNE_WIND_50 ASE_WIND_50 BSE_WIND_50 ASW_WIND_50 BSW_WIND_50 ANW_WIND_50 BNW_WIND_50 AAL_WIND_64 BAL_WIND_64 ANE_WIND_64 BNE_WIND_64 ASE_WIND_64 BSE_WIND_64 ASW_WIND_64 BSW_WIND_64 ANW_WIND_64 BNW_WIND_64 ARADP BRADP ARRP BRRP AMRD BMRD AGUSTS BGUSTS AEYE BEYE ADIR BDIR ASPEED BSPEED ADEPTH BDEPTH NUM_MEMBERS TRACK_SPREAD TRACK_STDEV MSLP_STDEV MAX_WIND_STDEV -TCST_TCDIAG : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE AMODEL BMODEL STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK TOTAL INDEX DIAG_SOURCE N_DIAG _VAR_ +TCST_TCDIAG : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE AMODEL BMODEL STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK TOTAL INDEX DIAG_SOURCE TRACK_SOURCE FIELD_SOURCE N_DIAG _VAR_ TCST_PROBRIRW : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE ALAT ALON BLAT BLON INITIALS TK_ERR X_ERR Y_ERR ADLAND BDLAND RI_BEG RI_END RI_WINDOW AWIND_END BWIND_BEG BWIND_END BDELTA BDELTA_MAX BLEVEL_BEG BLEVEL_END N_THRESH _VAR_ diff --git a/internal/test_unit/perl/tcst_conv.pl b/internal/test_unit/perl/tcst_conv.pl index 1ab4ec7b11..f1b9fd7122 100755 --- a/internal/test_unit/perl/tcst_conv.pl +++ b/internal/test_unit/perl/tcst_conv.pl @@ -33,7 +33,7 @@ () NUM_MEMBERS TRACK_SPREAD TRACK_STDEV MSLP_STDEV MAX_WIND_STDEV); my @fld_tcdiag = qw(AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK - TOTAL INDEX LEVEL DIAG_SOURCE N_DIAG DIAG_ VALUE_); + TOTAL INDEX LEVEL DIAG_SOURCE TRACK_SOURCE FIELD_SOURCE N_DIAG DIAG_ VALUE_); my @fld_probrirw = qw(AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK ALAT ALON BLAT BLON INITIALS TK_ERR X_ERR Y_ERR ADLAND BDLAND RI_BEG RI_END RI_WINDOW @@ -159,6 +159,8 @@ () "%15s" . # TOTAL "%15s" . # INDEX "%15s" . # DIAG_SOURCE + "%15s" . # TRACK_SOURCE + "%15s" . # FIELD_SOURCE "%15s"; # N_DIAG my $fmt_probrirw = @@ -361,9 +363,11 @@ () # 14 - TOTAL # 15 - INDEX # 16 - DIAG_SOURCE -# 17 - N_DIAG -# 18 - DIAG_i -# 19 - VALUE_i +# 17 - TRACK_SOURCE +# 18 - FIELD_SOURCE +# 19 - N_DIAG +# 20 - DIAG_i +# 21 - VALUE_i # PROBRIRW Line Type # 14 - ALAT diff --git a/internal/test_unit/xml/unit_tc_pairs.xml b/internal/test_unit/xml/unit_tc_pairs.xml index 0080eb53b1..24bec42c00 100644 --- a/internal/test_unit/xml/unit_tc_pairs.xml +++ b/internal/test_unit/xml/unit_tc_pairs.xml @@ -188,13 +188,14 @@ STORM_ID "AL092022" INIT_BEG "20220926_00" INIT_END "20220926_18" - DIAG_NAME "DTL", "PW01", "SHRD", "TPW", "LAND", "SHR_MAG", "STM_SPD" + CIRA_RT_DIAG_NAME "TPW", "LAND", "SHR_MAG", "STM_SPD" + SHIPS_RT_DIAG_NAME "DTL", "PW01", "SHRD" \ -adeck &DATA_DIR;/adeck/aal092022_OFCL_SHIP_AVNO.dat \ -bdeck &DATA_DIR;/bdeck/bal092022.dat \ - -diag TCDIAG &DATA_DIR;/tcdiag/2022/sal092022_avno_doper_20220926*_diag.dat \ - -diag LSDIAG_RT &DATA_DIR;/lsdiag_rt/2022/220926*AL0922_lsdiag.dat model=OFCL,SHIP \ + -diag CIRA_DIAG_RT &DATA_DIR;/diag/cira_diag_rt/2022/sal092022_avno_doper_20220926*_diag.dat \ + -diag SHIPS_DIAG_RT &DATA_DIR;/diag/ships_diag_rt/2022/220926*AL0922_lsdiag.dat \ -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ -out &OUTPUT_DIR;/tc_pairs/al092022_20220926_DIAGNOSTICS \ -log &OUTPUT_DIR;/tc_pairs/tc_pairs_DIAGNOSTICS.log \ diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 1141e2c75e..58701b893d 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -90,14 +90,28 @@ enum TrackType { // enum DiagType { - DiagType_None, // Default - TCDiagType, // Tropical Cyclone Diagnostics - LSDiagRTType, // Realtime Large Scale Diagnostics - LSDiagDevType // Development Large Scale Diagnostics + DiagType_None, // Default + DiagType_CIRA_RT, // Realtime CIRA Tropical Cyclone Diagnostics + DiagType_CIRA_Dev, // Developmental CIRA Tropical Cyclone Diagnostics + DiagType_SHIPS_RT, // Realtime SHIPS Large Scale Diagnostics + DiagType_SHIPS_Dev // Developmental SHIPS Large Scale Diagnostics }; //////////////////////////////////////////////////////////////////////// +// +// Corresponding diagnostic type strings +// + +static const char cira_diag_str[] = "CIRA_DIAG"; +static const char cira_diag_rt_str[] = "CIRA_DIAG_RT"; +static const char cira_diag_dev_str[] = "CIRA_DIAG_DEV"; +static const char ships_diag_str[] = "SHIPS_DIAG"; +static const char ships_diag_rt_str[] = "SHIPS_DIAG_RT"; +static const char ships_diag_dev_str[] = "SHIPS_DIAG_DEV"; + +//////////////////////////////////////////////////////////////////////// + // // Enumeration for 12-hour interpolation logic // @@ -1081,9 +1095,13 @@ static const char conf_key_dland_file[] = "dland_file"; static const char conf_key_basin_file[] = "basin_file"; static const char conf_key_track_watch_warn[] = "track_watch_warn"; static const char conf_key_watch_warn[] = "watch_warn"; +static const char conf_key_diag_info_map[] = "diag_info_map"; +static const char conf_key_diag_source[] = "diag_source"; +static const char conf_key_track_source[] = "track_source"; +static const char conf_key_field_source[] = "field_source"; +static const char conf_key_match_to_track[] = "match_to_track"; static const char conf_key_diag_name[] = "diag_name"; static const char conf_key_diag_convert_map[] = "diag_convert_map"; -static const char conf_key_source[] = "source"; static const char conf_key_basin_map[] = "basin_map"; static const char conf_key_time_offset[] = "time_offset"; static const char conf_key_amodel[] = "amodel"; diff --git a/src/basic/vx_config/config_util.cc b/src/basic/vx_config/config_util.cc index 607844db71..52817f1b6d 100644 --- a/src/basic/vx_config/config_util.cc +++ b/src/basic/vx_config/config_util.cc @@ -1121,7 +1121,7 @@ map parse_conf_key_convert_map( exit(1); } - // Conf: tcdiag_convert_map, lsdiag_convert_map, etc + // Conf: diag_convert_map map_dict = dict->lookup_array(conf_key_map_name); // Loop through the array entries @@ -2757,16 +2757,13 @@ ConcatString tracktype_to_string(TrackType type) { DiagType string_to_diagtype(const char *s) { DiagType t = DiagType_None; - // Convert string to enumerated DiagType - if(strcasecmp(s, conf_val_none) == 0) t = DiagType_None; - else if(strcasecmp(s, "TCDIAG") == 0) t = TCDiagType; - else if(strcasecmp(s, "LSDIAG_RT") == 0) t = LSDiagRTType; - else if(strcasecmp(s, "LSDIAG_DEV") == 0) t = LSDiagDevType; - else { - mlog << Error << "\nstring_to_diagtype() -> " - << "Unexpected DiagType string \"" << s << "\".\n\n"; - exit(1); - } + // Convert string to enumerated DiagType, storing unknown strings + // as the default type + if(strcasecmp(s, cira_diag_rt_str) == 0) t = DiagType_CIRA_RT; + else if(strcasecmp(s, cira_diag_dev_str) == 0) t = DiagType_CIRA_Dev; + else if(strcasecmp(s, ships_diag_rt_str) == 0) t = DiagType_SHIPS_RT; + else if(strcasecmp(s, ships_diag_dev_str) == 0) t = DiagType_SHIPS_Dev; + else t = DiagType_None; return(t); } @@ -2778,10 +2775,11 @@ ConcatString diagtype_to_string(DiagType type) { // Convert enumerated DiagType to string switch(type) { - case(DiagType_None): s = conf_val_none; break; - case(TCDiagType): s = "TCDIAG"; break; - case(LSDiagRTType): s = "LSDIAG_RT"; break; - case(LSDiagDevType): s = "LSDIAG_DEV"; break; + case(DiagType_None): s = conf_val_none; break; + case(DiagType_CIRA_RT): s = cira_diag_rt_str; break; + case(DiagType_CIRA_Dev): s = cira_diag_dev_str; break; + case(DiagType_SHIPS_RT): s = ships_diag_rt_str; break; + case(DiagType_SHIPS_Dev): s = ships_diag_dev_str; break; default: mlog << Error << "\ndiagtype_to_string() -> " << "Unexpected DiagType value of " << type << ".\n\n"; diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 32a37d2f07..3a4bced924 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -27,19 +27,17 @@ using namespace std; //////////////////////////////////////////////////////////////////////// -static const char default_lsdiag_technique[] = "OFCL"; - -static const int lsdiag_wdth[] = { +static const int ships_wdth[] = { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 }; -static int n_lsdiag_wdth = sizeof(lsdiag_wdth)/sizeof(*lsdiag_wdth); +static int n_ships_wdth = sizeof(ships_wdth)/sizeof(*ships_wdth); -static const int tcdiag_fill_value = 9999; -static const int lsdiag_fill_value = 9999; +static const int cira_fill_value = 9999; +static const int ships_fill_value = 9999; -static const char lsdiag_skip_str[] = "TIME,DELV,MTPW,IR00,IRM1,IRM3,PC00,PCM1,PCM3,PSLV,IRXX"; +static const char ships_skip_str[] = "TIME,DELV,MTPW,IR00,IRM1,IRM3,PC00,PCM1,PCM3,PSLV,IRXX"; //////////////////////////////////////////////////////////////////////// // @@ -167,7 +165,9 @@ void DiagFile::init_from_scratch() { void DiagFile::clear() { // Initialize values - SourceType = DiagType_None; + DiagSource = DiagType_None; + TrackSource.clear(); + FieldSource.clear(); StormId.clear(); Basin.clear(); Cyclone.clear(); @@ -209,30 +209,38 @@ void DiagFile::add_technique(const string &str) { //////////////////////////////////////////////////////////////////////// -void DiagFile::read(const DiagType source, - const ConcatString &path, const StringArray &model_names, - const std::map *convert_map) { +void DiagFile::read(const ConcatString &path, const ConcatString &diag_source, + const ConcatString &track_source, const ConcatString &field_source, + const StringArray &match_to_track, + const std::map *convert_map) { + + DiagType type = string_to_diagtype(diag_source.c_str()); - // Read diagnostics baesd on the source type - if(source == TCDiagType) { - read_tcdiag(path, model_names, convert_map); + // Read diagnostics based on the source type + if(type == DiagType_CIRA_RT) { + read_cira_rt(path, convert_map); } - else if(source == LSDiagRTType) { - read_lsdiag_rt(path, model_names, convert_map); + else if(type == DiagType_SHIPS_RT) { + read_ships_rt(path, convert_map); } else { mlog << Error << "\nDiagFile::read() -> " - << "diagnostics of type " << diagtype_to_string(source) - << " are not currently supported!\n\n"; + << "diagnostics of type \"" << diag_source + << "\" are not currently supported!\n\n"; exit(1); } + // Store the metadata + TrackSource = track_source; + FieldSource = field_source; + set_technique(match_to_track); + return; } //////////////////////////////////////////////////////////////////////// -void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_names, +void DiagFile::read_cira_rt(const ConcatString &path, const map *convert_map) { int i; double v_in, v_out; @@ -243,10 +251,7 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na clear(); // Store the file type - SourceType = TCDiagType; - - // Store user-specified model names or read it from the file below - if(model_names.n() > 0) set_technique(model_names); + DiagSource = DiagType_CIRA_RT; // Open the file open(path.c_str()); @@ -306,7 +311,7 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na // Check for the expected number of items if(NTime != LeadTime.n() || NTime != Lat.n() || NTime != Lon.n()) { - mlog << Error << "\nDiagFile::read_tcdiag() -> " + mlog << Error << "\nDiagFile::read_cira_rt() -> " << "the NTIME value (" << NTime << ") does not match the actual number of times (" << LeadTime.n() << "), latitudes (" << Lat.n() @@ -350,7 +355,7 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na v_in = atof(dl[i]); // Check for bad data and apply conversions - if(atoi(dl[i]) == tcdiag_fill_value) v_out = bad_data_double; + if(atoi(dl[i]) == cira_fill_value) v_out = bad_data_double; else if(fx_ptr) v_out = (*fx_ptr)(v_in); else v_out = v_in; @@ -360,7 +365,7 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na // Check for the expected number of items if(NTime != data.n()) { - mlog << Error << "\nDiagFile::read_tcdiag() -> " + mlog << Error << "\nDiagFile::read_cira_rt() -> " << "the number of \"" << cs << "\" diagnostic values (" << data.n() << ") does not match the expected number (" << NTime << "): " << path << "\n\n"; @@ -385,8 +390,8 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na //////////////////////////////////////////////////////////////////////// -void DiagFile::read_lsdiag_rt(const ConcatString &path, const StringArray &model_names, - const map *convert_map) { +void DiagFile::read_ships_rt(const ConcatString &path, + const map *convert_map) { int i, v_int; double v_dbl; NumArray data; @@ -396,18 +401,14 @@ void DiagFile::read_lsdiag_rt(const ConcatString &path, const StringArray &model clear(); // Store the file type - SourceType = LSDiagRTType; - - // Store user-specified model names or the default one - if(model_names.n() > 0) set_technique(model_names); - else add_technique(default_lsdiag_technique); + DiagSource = DiagType_SHIPS_RT; // Open the file open(path.c_str()); // Diagnostic names to ignore StringArray skip_diag_sa; - skip_diag_sa.parse_css(lsdiag_skip_str); + skip_diag_sa.parse_css(ships_skip_str); // Parse the header information from the first line DataLine dl; @@ -416,7 +417,7 @@ void DiagFile::read_lsdiag_rt(const ConcatString &path, const StringArray &model // Check for the expected number of items if(dl.n_items() != 9 || strncasecmp(dl[8], "HEAD", strlen("HEAD") != 0)) { - mlog << Error << "\nDiagFile::read_lsdiag_rt() -> " + mlog << Error << "\nDiagFile::read_ships_rt() -> " << "unexpected header line: " << path << "\n\n"; exit(1); } @@ -438,7 +439,7 @@ void DiagFile::read_lsdiag_rt(const ConcatString &path, const StringArray &model int data_start_location = in->tellg(); // Parse time and location info - while(read_fwf_line(dl, lsdiag_wdth, n_lsdiag_wdth)) { + while(read_fwf_line(dl, ships_wdth, n_ships_wdth)) { // Fixed width column 24 has the data name cs = dl[23]; @@ -473,7 +474,7 @@ void DiagFile::read_lsdiag_rt(const ConcatString &path, const StringArray &model in->seekg(data_start_location); // Store the diagnostics data - while(read_fwf_line(dl, lsdiag_wdth, n_lsdiag_wdth)) { + while(read_fwf_line(dl, ships_wdth, n_ships_wdth)) { // Skip empty lines if(dl.n_items() == 0) continue; @@ -506,7 +507,7 @@ void DiagFile::read_lsdiag_rt(const ConcatString &path, const StringArray &model v_int = atoi(dl[i]); // Check for bad data and apply conversions - if(v_int == lsdiag_fill_value) v_dbl = bad_data_double; + if(v_int == ships_fill_value) v_dbl = bad_data_double; else if(fx_ptr) v_dbl = (*fx_ptr)(v_int); else v_dbl = (double) v_int; @@ -516,7 +517,7 @@ void DiagFile::read_lsdiag_rt(const ConcatString &path, const StringArray &model // Check for the expected number of items if(NTime != data.n()) { - mlog << Error << "\nDiagFile::read_lsdiag_rt() -> " + mlog << Error << "\nDiagFile::read_ships_rt() -> " << "the number of \"" << cs << "\" diagnostic values (" << data.n() << ") does not match the expected number (" << NTime << ")!\n\n"; diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 6db6848870..c9109f9475 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -23,7 +23,7 @@ //////////////////////////////////////////////////////////////////////// // -// TCDIAG files: +// Realtime CIRA Diagnostics files: // - Add link to sample data // - Header: // * ATCF_ID YYYYMMDDHH * @@ -33,7 +33,7 @@ // BB is the 2-letter basin name // CC is the 2-digit cyclone number // -// Real-time LSDIAG files: +// Real-time SHIPS Diagnostics files: // - https://ftp.nhc.noaa.gov/atcf/lsdiag // - Header: // BBCC YYMMDD HH WS LAT LON 9999 BBCCYYYY @@ -42,7 +42,7 @@ // YYMMDD HH is the initialization time // WS is the wind speed // -// Developmental LSDIAG files (not currently supported): +// Developmental SHIPS Diagnostics files (not currently supported): // - https://rammb2.cira.colostate.edu/research/tropical-cyclones/ships // //////////////////////////////////////////////////////////////////////// @@ -58,8 +58,10 @@ class DiagFile : public LineDataFile { void init_from_scratch(); - // Diagnostics file type - DiagType SourceType; + // Diagnostics Metadata + DiagType DiagSource; + ConcatString TrackSource; + ConcatString FieldSource; // Storm and model identification ConcatString StormId; @@ -108,7 +110,9 @@ class DiagFile : public LineDataFile { double lat(int) const; double lon(int) const; - DiagType source() const; + DiagType diag_source() const; + const ConcatString & track_source() const; + const ConcatString & field_source() const; int n_diag() const; const StringArray & diag_name() const; bool has_diag(const std::string &) const; @@ -118,27 +122,30 @@ class DiagFile : public LineDataFile { // do stuff // - void read (const DiagType, - const ConcatString &, const StringArray &, - const std::map *); - void read_tcdiag (const ConcatString &, const StringArray &, - const std::map *); - void read_lsdiag_rt (const ConcatString &, const StringArray &, - const std::map *); + void read (const ConcatString &, const ConcatString &, + const ConcatString &, const ConcatString &, + const StringArray &, + const std::map *); + void read_cira_rt (const ConcatString &, + const std::map *); + void read_ships_rt (const ConcatString &, + const std::map *); }; //////////////////////////////////////////////////////////////////////// -inline const ConcatString & DiagFile::storm_id() const { return(StormId); } -inline const ConcatString & DiagFile::basin() const { return(Basin); } -inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); } -inline const StringArray & DiagFile::technique() const { return(Technique); } -inline unixtime DiagFile::init() const { return(InitTime); } -inline int DiagFile::n_time() const { return(NTime); } -inline DiagType DiagFile::source() const { return(SourceType); } -inline int DiagFile::n_diag() const { return(DiagName.n()); } -inline const StringArray & DiagFile::diag_name() const { return(DiagName); } +inline const ConcatString & DiagFile::storm_id() const { return(StormId); } +inline const ConcatString & DiagFile::basin() const { return(Basin); } +inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); } +inline const StringArray & DiagFile::technique() const { return(Technique); } +inline unixtime DiagFile::init() const { return(InitTime); } +inline int DiagFile::n_time() const { return(NTime); } +inline DiagType DiagFile::diag_source() const { return(DiagSource); } +inline const ConcatString & DiagFile::track_source() const { return(TrackSource); } +inline const ConcatString & DiagFile::field_source() const { return(FieldSource); } +inline int DiagFile::n_diag() const { return(DiagName.n()); } +inline const StringArray & DiagFile::diag_name() const { return(DiagName); } //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/tc_columns.cc b/src/libcode/vx_tc_util/tc_columns.cc index 6c6d997981..b561c448c9 100644 --- a/src/libcode/vx_tc_util/tc_columns.cc +++ b/src/libcode/vx_tc_util/tc_columns.cc @@ -360,6 +360,8 @@ void write_tc_diag_cols(const TrackPairInfo &p, int i, at.set_entry(r, c++, p.n_points()); at.set_entry(r, c++, i+1); at.set_entry(r, c++, diagtype_to_string(p.adeck().diag_source())); + at.set_entry(r, c++, p.adeck().track_source()); + at.set_entry(r, c++, p.adeck().field_source()); at.set_entry(r, c++, p.adeck()[i].n_diag()); // Check the number of names and values match diff --git a/src/libcode/vx_tc_util/tc_columns.h b/src/libcode/vx_tc_util/tc_columns.h index 522a66377d..8236e141df 100644 --- a/src/libcode/vx_tc_util/tc_columns.h +++ b/src/libcode/vx_tc_util/tc_columns.h @@ -120,8 +120,9 @@ static const int n_tc_cols_xy = sizeof(tc_cols_xy)/sizeof(*tc_cols_xy); //////////////////////////////////////////////////////////////////////// static const char * tc_diag_cols [] = { - "TOTAL", "INDEX", "DIAG_SOURCE", - "N_DIAG", "DIAG_", "VALUE_" + "TOTAL", "INDEX", + "DIAG_SOURCE", "TRACK_SOURCE", "FIELD_SOURCE", + "N_DIAG", "DIAG_", "VALUE_" }; static const int n_tc_diag_cols = sizeof(tc_diag_cols)/sizeof(*tc_diag_cols); diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index 5eec6f5b1d..f99d95cf3e 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -92,6 +92,8 @@ void TrackInfo::clear() { MinWarmCore = (unixtime) 0; MaxWarmCore = (unixtime) 0; DiagSource = DiagType_None; + TrackSource.clear(); + FieldSource.clear(); DiagName.clear(); TrackLines.clear(); @@ -133,6 +135,8 @@ void TrackInfo::dump(ostream &out, int indent_depth) const { out << prefix << "MinWarmCore = \"" << (MinWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MinWarmCore).text() : na_str) << "\n"; out << prefix << "MaxWarmCore = \"" << (MaxWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MaxWarmCore).text() : na_str) << "\n"; out << prefix << "DiagSource = " << diagtype_to_string(DiagSource) << "\n"; + out << prefix << "TrackSource = " << TrackSource.contents() << "\n"; + out << prefix << "FieldSource = " << FieldSource.contents() << "\n"; out << prefix << "NDiag = " << DiagName.n() << "\n"; out << prefix << "NPoints = " << NPoints << "\n"; out << prefix << "NAlloc = " << NAlloc << "\n"; @@ -172,6 +176,8 @@ ConcatString TrackInfo::serialize() const { << ", MinWarmCore = " << (MinWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MinWarmCore).text() : na_str) << ", MaxWarmCore = " << (MaxWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MaxWarmCore).text() : na_str) << ", DiagSource = " << diagtype_to_string(DiagSource) + << ", TrackSource = " << TrackSource.contents() + << ", FieldSource = " << FieldSource.contents() << ", NDiag = " << DiagName.n() << ", NPoints = " << NPoints << ", NAlloc = " << NAlloc @@ -223,6 +229,8 @@ void TrackInfo::assign(const TrackInfo &t) { MinWarmCore = t.MinWarmCore; MaxWarmCore = t.MaxWarmCore; DiagSource = t.DiagSource; + TrackSource = t.TrackSource; + FieldSource = t.FieldSource; DiagName = t.DiagName; TrackLines = t.TrackLines; @@ -534,8 +542,10 @@ bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &req_diag_n InitTime != diag_file.init() || !diag_file.technique().has(Technique)) return(false); - // Store the diagnostic source - DiagSource = diag_file.source(); + // Store the diagnostic metadata + DiagSource = diag_file.diag_source(); + TrackSource = diag_file.track_source(); + FieldSource = diag_file.field_source(); // If empty, store all diagnostics bool store_all_diag = (req_diag_name.n() == 0 ? true : false); @@ -563,12 +573,21 @@ bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &req_diag_n if(i_name == 0) { if(!is_eq(diag_file.lat(i_time), Point[i_pnt].lat()) || !is_eq(diag_file.lon(i_time), Point[i_pnt].lon())) { - mlog << Warning << "\nTrackInfo::add_diag_data() -> " - << "the " << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) + ConcatString cs; + cs << "The " << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) << " lead time " << sec_to_timestring(diag_file.lead(i_time)) << " track location (" << Point[i_pnt].lat() << ", " - << Point[i_pnt].lon() << ") does not match the diagnostic location (" - << diag_file.lat(i_time) << ", " << diag_file.lon(i_time) << ")\n"; + << Point[i_pnt].lon() << ") does not match the " + << diagtype_to_string(DiagSource) << " diagnostic location (" + << diag_file.lat(i_time) << ", " << diag_file.lon(i_time) << ")"; + + // Print a warning if the TrackSource and Technique match + if(TrackSource == Technique) { + mlog << Warning << "\nTrackInfo::add_diag_data() -> " << cs << "\n\n"; + } + else { + mlog << Debug(4) << cs << "\n"; + } } } diff --git a/src/libcode/vx_tc_util/track_info.h b/src/libcode/vx_tc_util/track_info.h index 6cbe99e3e2..05b07b2f31 100644 --- a/src/libcode/vx_tc_util/track_info.h +++ b/src/libcode/vx_tc_util/track_info.h @@ -71,8 +71,10 @@ class TrackInfo { unixtime MaxWarmCore; // Diagnostic source and names - DiagType DiagSource; - StringArray DiagName; + DiagType DiagSource; + ConcatString TrackSource; + ConcatString FieldSource; + StringArray DiagName; // TrackPoints TrackPoint *Point; @@ -115,6 +117,8 @@ class TrackInfo { void set_valid_max(const unixtime); void set_point(int, const TrackPoint &); void set_diag_source(DiagType); + void set_track_source(const char *); + void set_field_source(const char *); void set_diag_name(const StringArray &); // @@ -148,6 +152,8 @@ class TrackInfo { int n_points() const; DiagType diag_source() const; + const ConcatString & track_source() const; + const ConcatString & field_source() const; int n_diag() const; const StringArray & diag_name() const; const char * diag_name(int) const; @@ -189,6 +195,8 @@ inline void TrackInfo::set_init(const unixtime u) { InitTime = u; inline void TrackInfo::set_valid_min(const unixtime u) { MinValidTime = u; } inline void TrackInfo::set_valid_max(const unixtime u) { MaxValidTime = u; } inline void TrackInfo::set_diag_source(DiagType t) { DiagSource = t; } +inline void TrackInfo::set_track_source(const char *s) { TrackSource = s; } +inline void TrackInfo::set_field_source(const char *s) { FieldSource = s; } inline void TrackInfo::set_diag_name(const StringArray &s) { DiagName = s; } inline const ConcatString & TrackInfo::storm_id() const { return(StormId); } @@ -207,6 +215,8 @@ inline unixtime TrackInfo::warm_core_max() const { return(MaxWarm inline int TrackInfo::n_points() const { return(NPoints); } inline DiagType TrackInfo::diag_source() const { return(DiagSource); } +inline const ConcatString & TrackInfo::track_source() const { return(TrackSource); } +inline const ConcatString & TrackInfo::field_source() const { return(FieldSource); } inline int TrackInfo::n_diag() const { return(DiagName.n()); } inline const StringArray & TrackInfo::diag_name() const { return(DiagName); } diff --git a/src/libcode/vx_tc_util/track_pair_info.cc b/src/libcode/vx_tc_util/track_pair_info.cc index 15ba978052..5be866340c 100644 --- a/src/libcode/vx_tc_util/track_pair_info.cc +++ b/src/libcode/vx_tc_util/track_pair_info.cc @@ -471,21 +471,43 @@ void TrackPairInfo::add_tcdiag_line(const TCStatLine &l) { // Name of diagnostics read StringArray diag_name; - // Diagnostic source type - DiagType t = string_to_diagtype(l.get_item("DIAG_SOURCE")); - - // Make sure the source type does not change + // Make sure DIAG_SOURCE does not change + DiagType diag_source = string_to_diagtype(l.get_item("DIAG_SOURCE")); if(ADeck.diag_source() != DiagType_None && - ADeck.diag_source() != t) { + ADeck.diag_source() != diag_source) { mlog << Error << "\nTrackPairInfo::add_tcdiag_line() -> " << "the diagnostic source type has changed (" << diagtype_to_string(ADeck.diag_source()) << " != " - << diagtype_to_string(t) << ")!\n\n"; + << diagtype_to_string(diag_source) << ")!\n\n"; + exit(1); + } + + // Make sure TRACK_SOURCE does not change + ConcatString track_source = l.get_item("TRACK_SOURCE"); + if(ADeck.track_source().length() > 0 && + ADeck.track_source() != track_source) { + mlog << Error << "\nTrackPairInfo::add_tcdiag_line() -> " + << "the diagnostic track source has changed (" + << ADeck.track_source() << " != " + << track_source << ")!\n\n"; + exit(1); + } + + // Make sure FIELD_SOURCE does not change + ConcatString field_source = l.get_item("FIELD_SOURCE"); + if(ADeck.field_source().length() > 0 && + ADeck.field_source() != field_source) { + mlog << Error << "\nTrackPairInfo::add_tcdiag_line() -> " + << "the diagnostic field source has changed (" + << ADeck.field_source() << " != " + << field_source << ")!\n\n"; exit(1); } - // Store the source type - ADeck.set_diag_source(t); + // Store the diagnostic metadata + ADeck.set_diag_source(diag_source); + ADeck.set_track_source(track_source.c_str()); + ADeck.set_field_source(field_source.c_str()); // Number of diagnostics n_diag = atoi(l.get_item("N_DIAG")); diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index 1861e16eb0..734f83d256 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -247,9 +247,8 @@ void process_command_line(int argc, char **argv) { for(i=0; i n_diag_map; int i, j, n; - map n_diag_map; - // Process the diagnostic inputs - for(i=0; i " + << "the diagnostic path and source arrays must have equal length!\n\n"; + exit(1); + } - // Process the current source - cur_path.clear(); - cur_path.add(diag_path[i]); - cur_model_name.clear(); - cur_model_name.add(diag_model_name[i]); + // Process the diagnostic inputs + for(i=0; i " + << "no \"" << conf_key_diag_info_map + << "\" entry found for diagnostic source \"" + << diag_source[i] << "\"!\n\n"; + exit(1); + } + info_ptr = &conf_info.DiagInfoMap.at(diag_source[i]); - // Pointer to the conversion map for this source + // Check for diagnostic conversion map entry map * convert_ptr = 0; - if(conf_info.DiagConvertMap.count(diag_source[i]) > 0) { - convert_ptr = &conf_info.DiagConvertMap.at(diag_source[i]); + map< ConcatString,map >::iterator it; + for(it = conf_info.DiagConvertMap.begin(); + it != conf_info.DiagConvertMap.end(); it++) { + + // Partial string matching for diag_source + if(check_reg_exp(it->first.c_str(), diag_source[i].c_str())) { + convert_ptr = &it->second; + break; + } } // Process the diagnostic files - diag_file.read(diag_source[i], files[j], model_names, convert_ptr); + diag_file.read(cur_files[j], diag_source[i], + info_ptr->TrackSource, info_ptr->FieldSource, + info_ptr->MatchToTrack, convert_ptr); // Add diagnostics data to existing tracks - n += tracks.add_diag_data(diag_file, conf_info.DiagName); + n += tracks.add_diag_data(diag_file, info_ptr->DiagName); } // end for j @@ -1058,11 +1072,11 @@ void process_diags(TrackInfoArray &tracks) { } // end for i // Print the diagnostic track counts - for(map::iterator it = n_diag_map.begin(); + for(map::iterator it = n_diag_map.begin(); it != n_diag_map.end(); it++) { mlog << Debug(3) - << "Added " << diagtype_to_string(it->first) - << " diagnostics to " << it->second << " tracks.\n"; + << "Added " << it->first << " diagnostics to " + << it->second << " tracks.\n"; } return; @@ -2233,8 +2247,8 @@ void usage() { << "\t\t\"-diag source path\" is used one or more times to " << "specify a file or top-level directory containing tropical " << "cyclone diagnostics \"" << atcf_suffix - << "\" data to process. The supported formats are TCDIAG, " - << "LSDIAG_RT, LSDIAG_DEV (optional).\n" + << "\" data to process. The supported sources are CIRA_DIAG_RT " + << "and SHIPS_DIAG_RT (optional).\n" << "\t\t\"-out base\" overrides the default output file base " << "(" << out_base << ") (optional).\n" @@ -2247,11 +2261,7 @@ void usage() { << "\tNote: The \"-adeck\", \"-edeck\", and \"-bdeck\" options " << "may include \"suffix=string\" to modify the model names " - << "from that path.\n\n" - - << "\tNote: The \"-diag\" option may include \"model=string\" to " - << "override the model name of the tracks to which those diagnostics " - << "correspond.\n\n"; + << "from that path.\n\n"; exit(1); } @@ -2313,10 +2323,6 @@ void set_atcf_path(const StringArray &a, //////////////////////////////////////////////////////////////////////// void set_diag(const StringArray &a) { - int i; - StringArray sa; - DiagType source; - ConcatString cs, model_name; // Should have length of at least 2 if(a.n() < 2) { @@ -2326,33 +2332,10 @@ void set_diag(const StringArray &a) { exit(1); } - // Parse the DiagType source - source = string_to_diagtype(a[0].c_str()); - - // Check for the model sub-argument - for(i=1; i " - << "the model name must be specified as " - << "\"model=string\".\n\n"; - usage(); - } - else { - model_name = sa[1]; - } - } - } - // Parse the remaining paths - for(i=1; i diag_source; -static StringArray diag_path, diag_model_name; +static StringArray diag_source, diag_path; static ConcatString config_file; static TCPairsConfInfo conf_info; diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc index fa6c9b318c..b4faab8aef 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc @@ -25,8 +25,11 @@ using namespace std; //////////////////////////////////////////////////////////////////////// +void parse_conf_diag_info_map(Dictionary *, + map &); + void parse_conf_diag_convert_map(Dictionary *, - map > &); + map< ConcatString,map > &); //////////////////////////////////////////////////////////////////////// // @@ -101,7 +104,7 @@ void TCPairsConfInfo::clear() { DLandFile.clear(); WatchWarnFile.clear(); WatchWarnOffset = bad_data_int; - DiagName.clear(); + DiagInfoMap.clear(); DiagConvertMap.clear(); BasinMap.clear(); Version.clear(); @@ -312,8 +315,8 @@ void TCPairsConfInfo::process_config() { // Conf: WatchWarnOffset WatchWarnOffset = dict->lookup_int(conf_key_time_offset); - // Conf: DiagName - DiagName = dict->lookup_string_array(conf_key_diag_name); + // Conf: DiagInfoMap + parse_conf_diag_info_map(dict, DiagInfoMap); // Conf: DiagConvertMap parse_conf_diag_convert_map(dict, DiagConvertMap); @@ -330,14 +333,74 @@ void TCPairsConfInfo::process_config() { // //////////////////////////////////////////////////////////////////////// +void DiagInfo::clear() { + + TrackSource.clear(); + FieldSource.clear(); + MatchToTrack.clear(); + DiagName.clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void parse_conf_diag_info_map(Dictionary *dict, map &source_map) { + int i; + Dictionary *map_dict = (Dictionary *) 0; + ConcatString diag_source; + DiagInfo cur_info; + + const char *method_name = "parse_conf_diag_info_map() -> "; + + if(!dict) { + mlog << Error << "\n" << method_name + << "empty dictionary!\n\n"; + exit(1); + } + + // Conf: diag_info_map + map_dict = dict->lookup_array(conf_key_diag_info_map); + + // Loop through the array entries + for(i=0; in_entries(); i++) { + + // Initialize the current map + cur_info.clear(); + + // Lookup the metadata entries + diag_source = (*map_dict)[i]->dict_value()->lookup_string(conf_key_diag_source); + cur_info.TrackSource = (*map_dict)[i]->dict_value()->lookup_string(conf_key_track_source); + cur_info.FieldSource = (*map_dict)[i]->dict_value()->lookup_string(conf_key_field_source); + cur_info.MatchToTrack = (*map_dict)[i]->dict_value()->lookup_string_array(conf_key_match_to_track); + cur_info.DiagName = (*map_dict)[i]->dict_value()->lookup_string_array(conf_key_diag_name); + + // diag_source entries must be unique + if(source_map.count(diag_source) > 0) { + mlog << Error << "\n" << method_name + << "multiple entries found for diag_source \"" << diag_source + << "\" in \"" << conf_key_diag_info_map << "\"!\n\n"; + exit(1); + } + // Add a new source entry + else { + source_map[diag_source] = cur_info; + } + + } // end for i + + return; +} + +//////////////////////////////////////////////////////////////////////// + void parse_conf_diag_convert_map(Dictionary *dict, - map > &source_map) { + map< ConcatString,map > &source_map) { int i, j; Dictionary *map_dict = (Dictionary *) 0; map cur_map; - DiagType source; + ConcatString diag_source, key; StringArray sa; - ConcatString key; UserFunc_1Arg fx; const char *method_name = "parse_conf_diag_convert_map() -> "; @@ -358,9 +421,8 @@ void parse_conf_diag_convert_map(Dictionary *dict, cur_map.clear(); // Lookup the source, key, and convert function - source = string_to_diagtype( - (*map_dict)[i]->dict_value()->lookup_string(conf_key_source).c_str()); - sa = (*map_dict)[i]->dict_value()->lookup_string_array(conf_key_key); + diag_source = (*map_dict)[i]->dict_value()->lookup_string(conf_key_diag_source); + sa = (*map_dict)[i]->dict_value()->lookup_string_array(conf_key_key); fx.clear(); fx.set((*map_dict)[i]->dict_value()->lookup(conf_key_convert)); @@ -378,15 +440,15 @@ void parse_conf_diag_convert_map(Dictionary *dict, } // Append to the existing source entry - if(source_map.count(source) > 0) { + if(source_map.count(diag_source) > 0) { for(map::iterator it = cur_map.begin(); it != cur_map.end(); it++) { - source_map.at(source).insert(pair(it->first, it->second)); + source_map.at(diag_source).insert(pair(it->first, it->second)); } } // Add a new source entry else { - source_map.insert(pair >(source, cur_map)); + source_map.insert(pair< ConcatString,map >(diag_source, cur_map)); } } // end for i diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h index 12dc7eea14..bd65d31d45 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h @@ -33,6 +33,16 @@ struct ConsensusInfo { //////////////////////////////////////////////////////////////////////// +struct DiagInfo { + ConcatString TrackSource; + ConcatString FieldSource; + StringArray MatchToTrack; + StringArray DiagName; + void clear(); +}; + +//////////////////////////////////////////////////////////////////////// + class TCPairsConfInfo { private: @@ -111,11 +121,11 @@ class TCPairsConfInfo { // Watch/warnings time offset int WatchWarnOffset; - // Diagnostics to be extracted - StringArray DiagName; + // Diagnostics metadata + std::map DiagInfoMap; // Diagnostic conversions - std::map< DiagType, std::map > DiagConvertMap; + std::map< ConcatString,std::map > DiagConvertMap; // Basin Map std::map BasinMap;