generated from NOAA-OWP/owp-open-source-project-template
-
Notifications
You must be signed in to change notification settings - Fork 63
Realization Config
Justin Singh-M. - NOAA edited this page Nov 21, 2023
·
27 revisions
Table of Contents
The realization configuration file is a user-defined JSON file representing the unit(s) of work for a model engine run. A unit of work for model engine is represented by a conceptual catchment, particularly the geographical boundary of a catchment: its divide.
Tip
We can also think of a unit of work as an area or region of interest.
Each unit of work is associated to a specific type of work - a formulation - that wraps a model implementation in a supported programming language. These formulations are dynamically loaded "plug-ins" specified in the realization configuration file.
Note
For information on defining formulations or modeling, refer to Formulations and BMI.
This representation of work aims to align with the overarching goal of model engine, and the NextGen Framework as a whole, of hetereogenous modeling, such that different types of work can be applied to units of work on an individual basis.
The model engine realization configuration files are structured to represent these concepts with the following components:
- "Global"/default formulation,
- Specific formulation(s),
- Temporal domain,
- and, additional options or operations, such as flow routing.
Now, to represent these components in JSON, we provide the following (pseudo) schema:
{
// (1) Default Formulation
"global": {
"formulations": ["<single-bmi> | <multi-bmi>"],
"forcing": {}
},
// (2) Specific Formulation(s)
"catchments": {
"<id-1>": {
"formulations": ["<single-bmi> | <multi-bmi>"],
"forcing": {}
},
"<id-2>": {
"formulations": ["<single-bmi> | <multi-bmi>"],
"forcing": {}
}
},
// (3) Temporal Domain
"time": {
"start_time": "timestamp",
"end_time": "timestamp",
"output_interval": 0
},
// (4) Additional Options/Operations
"output_root": "path",
"routing": {
"t_route_config_file_with_path": "path"
}
}Note
The <single-bmi> and <multi-bmi> reference schemas are located in the Formulations and BMI documentation.
Each of these scenarios will use the same hydrofabric data source, representing Gage 01073000. The GeoPackage file corresponding to this example is located in data/gauge_01073000 in the repository. This example contains the following network:
graph LR;
cat-11410 --> |wb-11410| nex-11223;
cat-11223 --> |wb-11223| nex-11224;
cat-11371 --> |wb-11371| nex-11224;
cat-11509 --> |wb-11509| nex-11224;
cat-11224 --> |wb-11224| nex-11225;
nex-11223 --> |wb-11223| cat-11223;
nex-11224 --> |wb-11224| cat-11224;
{"type":"Topology","objects":{"collection":{"type":"GeometryCollection","name":"vsistdout","crs":{"type":"name","properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"}},"geometries":[{"type":"Polygon","arcs":[[7,8,9,10,11,12]]},{"type":"Polygon","arcs":[[13,14,-12]]},{"type":"Polygon","arcs":[[15,-14,-11]]},{"type":"Polygon","arcs":[[-8,16,17]]},{"type":"Polygon","arcs":[[-9,-18,18]]},{"type":"LineString","arcs":[0,1,2]},{"type":"LineString","arcs":[3]},{"type":"LineString","arcs":[4]},{"type":"LineString","arcs":[5]},{"type":"LineString","arcs":[6]},{"type":"Point","coordinates":[3028,4489]},{"type":"Point","coordinates":[7937,6583]},{"type":"Point","coordinates":[8079,4054]}]}},"arcs":[[[2997,6883],[-22,-49],[-120,-108],[-152,-87],[-134,-154],[-98,0],[-90,-90],[-30,-21],[25,-245],[19,-122],[136,-367],[73,-381],[8,-7],[-3,-70],[35,-150],[27,-60],[117,-122],[120,-196],[109,-137],[11,-28]],[[3028,4489],[43,-3],[74,-35],[46,-56],[24,-42],[19,-109],[27,-38],[125,-46],[30,-45],[19,-63],[28,-49],[35,-25],[46,-14],[161,46],[204,-50],[133,-49],[383,-238],[286,-81]],[[4711,3592],[153,290],[90,255],[109,86],[13,7],[92,46],[68,70],[49,56],[90,24],[5,14],[-5,21],[-35,56],[2,28],[63,14],[114,4],[63,-17],[60,-53],[57,28],[43,-3],[41,-21],[95,14],[36,25],[35,42],[16,160],[19,32],[123,115],[57,-20],[27,-39],[46,-7],[30,-14],[25,11],[11,31],[-14,133],[13,45],[33,74],[19,66],[0,25],[-8,17],[-9,84],[30,73],[22,130],[51,248],[74,94],[8,28],[32,84],[6,98],[11,21],[5,59],[-22,35],[-32,25],[-3,21],[95,73],[33,53],[125,129],[13,95],[14,10],[38,-3],[54,-21],[71,-21],[22,-21],[54,-21],[14,-14],[11,-28],[38,4],[73,56],[44,24],[101,14],[43,18],[44,49],[117,35],[51,-7],[71,7],[71,39],[65,0],[71,-21],[49,-73],[36,0]],[[3725,3321],[-168,60],[-207,42],[-51,-14],[-38,-28],[-36,-49],[-52,-129],[-103,-111],[-30,-74],[-36,-227],[-3,-181],[36,-98],[21,-35],[68,-49],[126,-36],[242,-14],[92,56],[57,56],[63,3],[22,-17],[19,-39],[13,-49],[3,-11],[59,-122],[102,-35],[192,93],[357,-82],[185,135],[23,111],[-43,92],[-7,17],[-22,49],[-67,105],[-38,119],[-14,101],[3,87],[104,308],[43,56],[71,146],[0,35]],[[7758,9548],[-83,8],[-52,-14],[-49,-60],[-19,-49],[-2,-122],[-11,-45],[-16,-35],[5,-81],[19,-45],[88,-66],[27,-42],[0,-88],[8,-28],[8,-101],[33,-52],[8,-25],[8,-10],[0,-21],[17,-63],[57,-105],[27,-59],[17,-38],[19,-21],[3,-28],[10,-14],[3,-28],[11,-14],[33,-14],[35,-101],[19,-95],[3,-84],[17,-63],[41,-55],[43,-119],[-2,-42],[-11,-52],[-82,-270],[-11,-84],[-38,-45],[-68,-122],[-5,-63],[0,-140],[27,-63],[39,-52],[0,-46],[-33,-21],[-11,-24],[-8,-46],[19,-56],[25,-31],[11,-31]],[[855,7559],[30,-41],[57,-154],[76,-234],[139,-287],[79,-129],[79,-192],[46,-158],[2,-76],[93,-224],[8,-74],[5,-429],[14,-130],[119,-188],[52,-158],[168,-391],[33,-105],[168,-290],[52,-28],[122,-130],[98,-45],[166,-57],[87,-59],[139,3],[90,73],[76,179],[175,254]],[[7937,6583],[40,-21],[11,-25],[6,-17],[16,-25],[63,-59],[-6,-31],[-38,-56],[3,-42],[11,-18],[-5,-133],[11,-31],[10,-14],[0,-17],[-5,-35],[-27,-35],[-41,-25],[-22,-17],[0,-147],[28,-56],[8,-56],[19,-21],[41,-31],[11,-28],[8,-63],[19,-45],[95,-84],[39,-63],[13,-38],[11,-182],[25,-119],[11,-171],[-16,-35],[5,-66],[44,-7],[30,-25],[-3,-70],[11,-70],[11,-10],[-6,-49],[-10,-31],[-19,-25],[-47,-28],[-79,-14],[-51,-21],[-30,-70],[-30,-24],[-6,-35],[28,-98],[22,-35],[40,-39],[0,-31],[-10,-14],[-68,-14],[-29,-13]],[[6399,7153],[219,-75],[119,87],[102,8],[33,158],[269,163],[125,-42],[37,-141],[94,-32],[168,-312],[241,-126],[114,-251]],[[7920,6590],[-73,-188],[-68,-317],[-119,-87],[-297,17],[-156,54],[-159,-116],[-2,-169],[99,-162]],[[7145,5622],[-90,-267],[-79,-58],[-144,-205],[-36,-328],[-127,-127],[-133,3],[-136,-166],[-258,46],[-76,-357],[-166,-155],[-76,-357],[-56,-108]],[[5768,3543],[-94,32],[-190,-105],[-81,-227],[-167,-156],[-187,64],[-40,-29],[-37,141],[-125,42],[42,198],[-187,64],[-102,-7],[-158,-116],[-343,117],[-128,-126],[-162,183],[-133,3],[-437,149],[-39,-29]],[[3200,3741],[-46,101],[-156,53],[101,476],[-77,111],[-204,-15],[-249,85],[-86,72],[-51,230],[-227,35],[-60,190],[19,249],[-187,64],[-6,130],[-86,71],[11,209],[-35,309],[-60,191],[42,198],[-62,22],[-69,150],[59,278],[-108,122],[-374,128],[-109,122],[-91,201],[-29,180],[-218,74],[-123,212],[-51,230],[-125,43]],[[543,8262],[-6,130],[-77,111],[181,66],[34,158],[-38,140],[-162,183],[-179,104],[-91,201],[127,127],[65,148],[187,-64],[15,-90],[201,-154],[94,-32],[150,76],[153,246],[93,-32],[37,-140],[281,-96],[140,-133],[91,-201],[247,-255],[94,-32],[179,-103],[181,65],[133,-3],[3,170],[119,86],[181,66],[200,314],[117,-82],[120,-381],[-3,-169],[69,-151],[-42,-199],[168,-312],[-59,-278],[46,-100],[-42,-198],[-119,-87],[-65,-148],[77,-112],[-34,-158],[52,-230],[-50,-238],[-102,-8],[-144,-206],[296,-654],[85,-71],[125,-43],[48,69],[204,15],[375,-128],[172,26],[119,87],[94,-32],[127,126],[143,674],[158,116],[26,119],[118,87],[96,137],[150,76],[26,119],[119,87],[54,-61],[312,-107],[110,47],[94,-32],[150,76],[33,159]],[[3200,3741],[-203,-483],[-135,-166],[-34,-159],[-150,-76],[-94,32],[-87,-98],[-148,93],[-79,-58],[-42,-198],[-312,107],[-181,-66],[-59,-278],[-39,-29]],[[1637,2362],[-54,61],[-38,141],[-116,82],[-14,90],[-69,151],[-147,93],[-123,212],[-29,179],[25,119],[-60,191],[-93,32],[-29,180],[-77,111],[-6,130],[-139,132],[-91,201],[-210,115],[50,237],[110,48],[34,158],[-60,191],[-154,222],[-247,255],[-100,161],[25,119],[277,203],[142,37],[94,-32],[41,198],[-99,162],[-29,180],[-94,32],[-20,219],[79,58],[133,-3],[79,58],[68,317],[-92,201],[-208,284],[51,238],[96,137]],[[5768,3543],[99,-162],[-34,-158],[83,-241],[63,-21],[-20,-249],[77,-111],[94,-32],[20,-220],[-136,-166],[-187,64],[-87,-98],[-82,-227],[-249,86],[-102,-8],[-184,-235],[-62,22],[-396,-290],[-59,-277],[-158,-116],[-192,-275],[-79,-58],[-42,-198],[60,-190],[-42,-199],[-96,-137],[-111,-47],[-139,133],[-91,201],[-52,230],[-118,-87],[-165,14],[-45,100],[-156,54],[-106,291],[167,155],[13,379],[-156,53],[-48,-69],[-133,3],[-312,107],[-100,162],[-124,42],[-77,111],[-156,54],[-69,151],[-124,42],[-77,112],[-187,63],[-54,61]],[[6399,7153],[-45,101],[75,356],[-31,11],[-66,320],[59,278],[119,87],[260,123],[134,-3],[127,126],[59,278],[-83,241],[-77,111],[280,372],[136,166],[190,105],[96,137],[141,37],[188,-64],[51,-230],[108,-122],[148,-93],[108,-123],[-136,-166],[-11,-209],[177,-273],[-34,-158],[-96,-137],[187,-64],[142,36],[20,-219],[145,-263],[117,-82],[99,-162]],[[8986,7670],[-56,-108],[-51,-238],[-204,-15],[-50,-238],[-357,-260],[-71,-19],[-277,-202]],[[8986,7670],[213,55],[87,97],[165,-13],[116,-83],[91,-201],[148,-93],[187,-65],[6,-129],[-142,-37],[-198,-144],[-125,43],[-79,-58],[-104,-177],[-45,-368],[62,-21],[83,-241],[-76,-357],[179,-103],[37,-141],[139,-132],[-135,-167],[-150,-76],[-160,-753],[-144,-205],[-134,3],[29,-180],[-33,-159],[-338,-12],[-187,64],[-144,-205],[-125,43],[-147,93],[-52,230],[-156,53],[-28,180],[-173,-26],[-100,162],[79,58],[76,357],[-122,212],[-187,64],[-129,341],[-125,43]]],"bbox":[-71.05775929974494,43.12209979722296,-70.9430165966376,43.187527995813156],"transform":{"scale":[0.000011475417852519166,0.000006543474206440388],"translate":[-71.05775929974494,43.12209979722296]}}
Scenario: We want to perform rainfall-runoff modeling across all divides in our domain. We know most of our analysis performs well with TOPMODEL, but we want to approximate the National Water Model on some divides using CFE to measure the performance.
In this case, we will consider cat-11410 and cat-11224 as our divides that will use CFE. Since we want to use TOPMODEL for the remaining defaults, we will define its formulation in the global section of the realization config.
Requirements:
- Built TOPMODEL shared library, e.g.
/my_build_dir/topmodel/libtopmodel.so - TOPMODEL initialization configuration file
- Built CFE shared library, e.g.
/my_build_dir/cfe/libcfe.so - CFE initialization configuration files for
cat-11410andcat-11224
Realization Configuration JSON
{
"global": {
"formulations": [
{
"name": "bmi_c",
"params": {
"model_type_name": "topmodel",
"library_file": "/my_build_dir/topmodel/libtopmodel.so",
"init_config": "",
"registration_function": "register_bmi_topmodel",
"main_output_variable": "Q",
"allow_exceed_end_time": true,
"uses_forcing_file": false,
"variables_names_map": {
"water_potential_evaporation_flux": "potential_evapotranspiration",
"atmosphere_water__liquid_equivalent_precipitation_rate": "precip_rate"
},
"model_params": {
"name": "Catchment Calibration Data",
"inputs_file": "/my_build_dir/topmodel/tm_inputs.dat",
"subcat_file": "",
"params_files": "/my_build_dir/topmodel/tm_params.dat",
"topmod_output_file": "./tm_topmod.out",
"hydro_output_file": "./tm_hyd.out"
}
}
}
]
},
"catchments": {
"cat-11410": {
"formulations": [
{
"name": "bmi_c",
"params": {
"model_type_name": "bmi_cfe",
"library_file": "/my_build_dir/cfe/libcfe.so",
"init_config": "/my_build_dir/cfe/cat-11410_cfe_config.txt",
"registration_function": "register_bmi_cfe",
"main_output_variable": "Q_OUT",
"allow_exceed_end_time": true,
"uses_forcing_file": false,
"variables_names_map": {
"water_potential_evaporation_flux": "potential_evapotranspiration",
"atmosphere_water__liquid_equivalent_precipitation_rate": "precip_rate",
"atmosphere_air_water~vapor__relative_saturation": "SPFH_2maboveground",
"land_surface_air__temperature": "TMP_2maboveground",
"land_surface_wind__x_component_of_velocity": "UGRD_10maboveground",
"land_surface_wind__y_component_of_velocity": "VGRD_10maboveground",
"land_surface_radiation~incoming~longwave__energy_flux": "DLWRF_surface",
"land_surface_radiation~incoming~shortwave__energy_flux": "DSWRF_surface",
"land_surface_air__pressure": "PRES_surface"
}
}
}
]
},
"cat-11224": {
"formulations": [
{
"name": "bmi_c",
"params": {
"model_type_name": "bmi_cfe",
"library_file": "/my_build_dir/cfe/libcfe.so",
"init_config": "/my_build_dir/cfe/cat-11224_cfe_config.txt",
"registration_function": "register_bmi_cfe",
"main_output_variable": "Q_OUT",
"allow_exceed_end_time": true,
"uses_forcing_file": false,
"variables_names_map": {
"water_potential_evaporation_flux": "potential_evapotranspiration",
"atmosphere_water__liquid_equivalent_precipitation_rate": "precip_rate",
"atmosphere_air_water~vapor__relative_saturation": "SPFH_2maboveground",
"land_surface_air__temperature": "TMP_2maboveground",
"land_surface_wind__x_component_of_velocity": "UGRD_10maboveground",
"land_surface_wind__y_component_of_velocity": "VGRD_10maboveground",
"land_surface_radiation~incoming~longwave__energy_flux": "DLWRF_surface",
"land_surface_radiation~incoming~shortwave__energy_flux": "DSWRF_surface",
"land_surface_air__pressure": "PRES_surface"
}
}
}
]
}
},
"time": {
"start_time": "2012-05-01 00:00:00",
"end_time": "2012-05-02 23:00:00",
"output_interval": 3600
}
}Scenario: We want to simulate land surface states using Noah-OWP-Modular and pass the resulting outputs to CFE across all catchments in our domain.
Requirements:
- Built NOAH-OWP-Modular shared library, e.g.
/my_build_dir/noahowp/libsurface.so - NOAH-OWP initialization configuration files for each divide in the domain.
- Built CFE shared library, e.g.
/my_build_dir/cfe/libcfe.so - CFE initialization configuration files for each divide in the domain.
Realization Configuration JSON
{
"global": {
"formulations": [
{
"name": "bmi_multi",
"params": {
"model_type_name": "bmi_noahowp_cfe",
"main_output_variable": "Q_OUT",
"modules": [
{
"name": "bmi_fortran",
"params": {
"model_type_name": "bmi_noahowp",
"library_file": "/my_build_dir/noahowp/libsurfacebmi.so",
"forcing_file": "",
"init_config": "/my_build_dir/noahowp-init-{{id}}.input",
"main_output_variable": "QINSUR",
"variables_names_map": {
"PRCPNONC": "atmosphere_water__liquid_equivalent_precipitation_rate",
"Q2": "atmosphere_air_water~vapor__relative_saturation",
"SFCTMP": "land_surface_air__temperature",
"UU": "land_surface_wind__x_component_of_velocity",
"VV": "land_surface_wind__y_component_of_velocity",
"LWDN": "land_surface_radiation~incoming~longwave__energy_flux",
"SOLDN": "land_surface_radiation~incoming~shortwave__energy_flux",
"SFCPRS": "land_surface_air__pressure"
},
"uses_forcing_file": false
}
},
{
"name": "bmi_c",
"params": {
"model_type_name": "bmi_cfe",
"library_file": "/my_build_dir/cfe/libcfe.so",
"init_config": "/my_build_dir/cfe/{{id}}_cfe_config.txt",
"registration_function": "register_bmi_cfe",
"main_output_variable": "Q_OUT",
"allow_exceed_end_time": true,
"uses_forcing_file": false,
"variables_names_map": {
"water_potential_evaporation_flux": "ETRAN",
"atmosphere_air_water~vapor__relative_saturation": "SPFH_2maboveground",
"land_surface_air__temperature": "TMP_2maboveground",
"land_surface_wind__x_component_of_velocity": "UGRD_10maboveground",
"land_surface_wind__y_component_of_velocity": "VGRD_10maboveground",
"land_surface_radiation~incoming~longwave__energy_flux": "DLWRF_surface",
"land_surface_radiation~incoming~shortwave__energy_flux": "DSWRF_surface",
"land_surface_air__pressure": "PRES_surface"
}
}
}
]
}
}
],
"forcing": {
"file_pattern": ".*{{id}}.*.csv",
"path": "/my_forcing_dir/",
"provider": "CsvPerFeature"
}
},
"catchments": {},
"time": {
"start_time": "2012-05-01 00:00:00",
"end_time": "2012-05-02 23:00:00",
"output_interval": 3600
}
}Tutorial
Getting Started
Configuration
Technical References