Merge pull request #2179 from CliMA/gb/diagnostics_ready

Add support for online remapping

.buildkite/pipeline.yml

@@ -154,12 +154,6 @@ steps:
           --config_file $CONFIG_PATH/box_hydrostatic_balance_rhoe.yml
         artifact_paths: "box_hydrostatic_balance_rhoe/*"
 
-      - label: ":computer: Bomex box (ρe_tot)"
-        command: >
-          julia --color=yes --project=examples examples/hybrid/driver.jl
-          --config_file $CONFIG_PATH/bomex_box_rhoe.yml
-        artifact_paths: "bomex_box_rhoe/*"
-
       - label: ":computer: 3D density current"
         command: >
           julia --color=yes --project=examples examples/hybrid/driver.jl

config/default_configs/default_config.yml

@@ -257,4 +257,10 @@ log_params:
   value: false
 output_default_diagnostics:
   help: "Output the default diagnostics associated to the selected atmospheric model"
+  value: true
+netcdf_interpolate_z_over_msl:
+  help: "Interpolate diagnostics in the NetCDF files in such a way that `z` is the elevation from the sea level (as opposed to the elevation from the surface)"
+  value: false
+warn_allocations_diagnostics:
+  help: "When true, a dry-run for all the diagnostics is perform to check whether the functions allocate additional memory (which reduces performances)"
   value: false

config/default_configs/default_perf.yml

@@ -10,3 +10,4 @@ forcing: "held_suarez"
 precip_model: "0M"
 dt_save_to_sol: "Inf"
 rad: "allskywithclear"
+warn_allocations_diagnostics: true

config/model_configs/box_density_current_test.yml

@@ -14,3 +14,6 @@ config: "box"
 kappa_4: 1.0e7
 z_max: 6400.0
 job_id: "box_density_current_test"
+diagnostics:
+  - short_name: thetaa
+    period: 10secs

config/model_configs/box_hydrostatic_balance_rhoe.yml

@@ -4,3 +4,8 @@ hyperdiff: "false"
 dt: "20secs"
 perturb_initstate: false
 job_id: "box_hydrostatic_balance_rhoe"
+diagnostics:
+  - short_name: wa
+    period: 1days
+  - short_name: ua
+    period: 1days

config/model_configs/plane_agnesi_mountain_test_stretched.yml

@@ -15,3 +15,7 @@ z_max: 25000.0
 topography: "Agnesi"
 job_id: "plane_agnesi_mountain_test_stretched"
 toml: [toml/plane_agnesi_mountain_test_stretched.toml]
+netcdf_interpolate_z_over_msl: true
+diagnostics:
+  - short_name: wa
+    period: 4hours

config/model_configs/plane_agnesi_mountain_test_uniform.yml

@@ -14,3 +14,7 @@ z_max: 25000.0
 topography: "Agnesi"
 job_id: "plane_agnesi_mountain_test_uniform"
 toml: [toml/plane_agnesi_mountain_test_uniform.toml]
+netcdf_interpolate_z_over_msl: true
+diagnostics:
+  - short_name: wa
+    period: 4hours

config/model_configs/plane_density_current_test.yml

@@ -11,3 +11,6 @@ config: "plane"
 kappa_4: 1.0e7
 z_max: 6400.0
 job_id: "plane_density_current_test"
+diagnostics:
+  - short_name: thetaa
+    period: 900secs

config/model_configs/plane_schar_mountain_test_stretched.yml

@@ -16,3 +16,7 @@ z_max: 25000.0
 topography: "Schar"
 job_id: "plane_schar_mountain_test_stretched"
 toml: [toml/plane_schar_mountain_test_stretched.toml]
+netcdf_interpolate_z_over_msl: true
+diagnostics:
+  - short_name: wa
+    period: 4hours

config/model_configs/plane_schar_mountain_test_uniform.yml

@@ -15,3 +15,7 @@ z_max: 25000.0
 topography: "Schar"
 job_id: "plane_schar_mountain_test_uniform"
 toml: [toml/plane_schar_mountain_test_uniform.toml]
+netcdf_interpolate_z_over_msl: true
+diagnostics:
+  - short_name: wa
+    period: 4hours

config/model_configs/prognostic_edmfx_bomex_fixtke_box.yml

@@ -28,74 +28,3 @@ dt: "5secs"
 t_end: "6hours"
 dt_save_to_disk: "10mins"
 toml: [toml/prognostic_edmfx_box.toml]
-diagnostics:
-  - short_name: ts
-    period: 10mins
-  - short_name: ta
-    period: 10mins
-  - short_name: thetaa
-    period: 10mins
-  - short_name: ha
-    period: 10mins
-  - short_name: pfull
-    period: 10mins
-  - short_name: rhoa
-    period: 10mins
-  - short_name: ua
-    period: 10mins
-  - short_name: va
-    period: 10mins
-  - short_name: wa
-    period: 10mins
-  - short_name: hur
-    period: 10mins
-  - short_name: hus
-    period: 10mins
-  - short_name: clw
-    period: 10mins
-  - short_name: cli
-    period: 10mins
-  - short_name: hussfc
-    period: 10mins
-  - short_name: evspsbl
-    period: 10mins
-  - short_name: arup
-    period: 10mins
-  - short_name: waup
-    period: 10mins
-  - short_name: taup
-    period: 10mins
-  - short_name: thetaaup
-    period: 10mins
-  - short_name: haup
-    period: 10mins
-  - short_name: husup
-    period: 10mins
-  - short_name: hurup
-    period: 10mins
-  - short_name: clwup
-    period: 10mins
-  - short_name: cliup
-    period: 10mins
-  - short_name: rhoaen
-    period: 10mins
-  - short_name: waen
-    period: 10mins
-  - short_name: taen
-    period: 10mins
-  - short_name: thetaaen
-    period: 10mins
-  - short_name: haen
-    period: 10mins
-  - short_name: husen
-    period: 10mins
-  - short_name: huren
-    period: 10mins
-  - short_name: clwen
-    period: 10mins
-  - short_name: clien
-    period: 10mins
-  - short_name: tke
-    period: 10mins
-  - short_name: lmix
-    period: 10mins

config/model_configs/single_column_nonorographic_gravity_wave.yml

@@ -1,8 +1,8 @@
-dt_save_to_disk: "500secs"
+dt_save_to_disk: "400secs"
 initial_condition: "IsothermalProfile"
 t_end: "1500secs"
 config: "column"
 hyperdiff: "false"
-dt: "500secs"
+dt: "400secs"
 non_orographic_gravity_wave: true
 job_id: "single_column_nonorographic_gravity_wave"

config/model_configs/single_column_precipitation_test.yml

@@ -20,15 +20,22 @@ toml: [toml/single_column_precipitation_test.toml]
 diagnostics:
   - short_name: hus
     period: 500secs
+    writer: h5
   - short_name: clw
     period: 500secs
+    writer: h5
   - short_name: cli
     period: 500secs
+    writer: h5
   - short_name: husra
     period: 500secs
+    writer: h5
   - short_name: hussn
     period: 500secs
+    writer: h5
   - short_name: ta
     period: 500secs
+    writer: h5
   - short_name: wa
     period: 500secs
+    writer: h5

config/model_configs/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml

@@ -1,11 +1,11 @@
 rad: "allskywithclear"
-dt_save_to_disk: "18hours"
+dt_save_to_disk: "1days"
 rayleigh_sponge: true
 orographic_gravity_wave: "raw_topo"
 z_elem: 25
 dt: "400secs"
 surface_setup: "DefaultMoninObukhov"
-t_end: "18hours"
+t_end: "15hours"
 non_orographic_gravity_wave: true
 dz_bottom: 300.0
 vert_diff: "true"
@@ -17,3 +17,4 @@ surface_temperature: "ZonallyAsymmetric"
 job_id: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric"
 moist: "equil"
 toml: [toml/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.toml]
+netcdf_interpolate_z_over_msl: true

config/model_configs/sphere_baroclinic_wave_rhoe.yml

@@ -3,3 +3,12 @@ initial_condition: "DryBaroclinicWave"
 dt: "400secs"
 t_end: "10days"
 job_id: "sphere_baroclinic_wave_rhoe"
+diagnostics:
+  - short_name: pfull
+    period: 1days
+  - short_name: wa
+    period: 1days
+  - short_name: va
+    period: 1days
+  - short_name: rv
+    period: 1days

config/model_configs/sphere_baroclinic_wave_rhoe_equilmoist.yml

@@ -5,3 +5,14 @@ initial_condition: "MoistBaroclinicWave"
 dt: "450secs"
 job_id: "sphere_baroclinic_wave_rhoe_equilmoist"
 moist: "equil"
+diagnostics:
+  - short_name: pfull
+    period: 1days
+  - short_name: wa
+    period: 1days
+  - short_name: va
+    period: 1days
+  - short_name: rv
+    period: 1days
+  - short_name: hus
+    period: 1days

config/model_configs/sphere_baroclinic_wave_rhoe_equilmoist_conservation_ft64.yml

@@ -7,3 +7,14 @@ dt: "450secs"
 FLOAT_TYPE: "Float64"
 job_id: "sphere_baroclinic_wave_rhoe_equilmoist_conservation_ft64"
 rad: "clearsky"
+diagnostics:
+  - short_name: pfull
+    period: 1days
+  - short_name: wa
+    period: 1days
+  - short_name: va
+    period: 1days
+  - short_name: rv
+    period: 1days
+  - short_name: hus
+    period: 1days

config/model_configs/sphere_baroclinic_wave_rhoe_equilmoist_edmfx.yml

@@ -1,8 +1,8 @@
 dt_save_to_disk: "30secs"
 initial_condition: "MoistBaroclinicWaveWithEDMF"
 dt: "1secs"
-edmfx_entr_model: "Generalized" 
-edmfx_detr_model: "Generalized" 
+edmfx_entr_model: "Generalized"
+edmfx_detr_model: "Generalized"
 edmfx_nh_pressure: true
 t_end: "6mins"
 turbconv: "edmfx"

config/model_configs/sphere_baroclinic_wave_rhoe_equilmoist_expvdiff.yml

@@ -9,3 +9,18 @@ t_end: "6hours"
 job_id: "sphere_baroclinic_wave_rhoe_equilmoist_expvdiff"
 moist: "equil"
 toml: [toml/sphere_baroclinic_wave_rhoe_equilmoist_expvdiff.toml]
+diagnostics:
+  - short_name: pfull
+    period: 12hours
+  - short_name: rv
+    period: 12hours
+  - short_name: wa
+    period: 12hours
+  - short_name: va
+    period: 12hours
+  - short_name: pfull
+    period: 12hours
+  - short_name: ta
+    period: 12hours
+  - short_name: hus
+    period: 12hours

config/model_configs/sphere_baroclinic_wave_rhoe_topography_dcmip_rs.yml

@@ -7,3 +7,13 @@ t_end: "6days"
 use_reference_state: false
 dt: "200secs"
 job_id: "sphere_baroclinic_wave_rhoe_topography_dcmip_rs"
+netcdf_interpolate_z_over_msl: true
+diagnostics:
+  - short_name: pfull
+    period: 1days
+  - short_name: wa
+    period: 1days
+  - short_name: va
+    period: 1days
+  - short_name: rv
+    period: 1days

config/model_configs/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml

@@ -13,4 +13,4 @@ viscous_sponge: true
 job_id: "sphere_held_suarez_rhoe_equilmoist_hightop_sponge"
 moist: "equil"
 toml: [toml/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.toml]
-output_default_diagnostics: true
+netcdf_interpolate_z_over_msl: true

config/model_configs/sphere_held_suarez_rhoe_equilmoist_topography_dcmip.yml

@@ -10,3 +10,4 @@ precip_model: "0M"
 topography: "DCMIP200"
 job_id: "sphere_held_suarez_rhoe_equilmoist_topography_dcmip"
 moist: "equil"
+netcdf_interpolate_z_over_msl: true

config/model_configs/sphere_held_suarez_rhoe_topography_dcmip.yml

@@ -8,3 +8,4 @@ forcing: "held_suarez"
 dt: "200secs"
 viscous_sponge: true
 job_id: "sphere_held_suarez_rhoe_topography_dcmip"
+netcdf_interpolate_z_over_msl: true

docs/src/diagnostics.md

@@ -28,7 +28,7 @@ identifies the period over which to compute the reduction and how often to save
 to disk. `output_name` is optional, and if provided, it identifies the name of the
 output file.
 
-The default `writer` is HDF5. If `writer` is `nc` or `netcdf`, the output is
+The default `writer` is NetCDF. If `writer` is `nc` or `netcdf`, the output is
 remapped non-conservatively on a Cartesian grid and saved to a NetCDF file.
 Currently, only 3D fields on cubed spheres are supported.
 
@@ -90,7 +90,8 @@ and time, or can be the result of a reduction in a period that is defined by
 More specifically, a `ScheduledDiagnostic` contains the following pieces of data
 
 - `variable`: The diagnostic variable that has to be computed and output.
-- `output_every`: Frequency of how often to save the results to disk.
+- `output_every`: Frequency of how often to save the results to disk. If
+  `output_every` is non-positive, only output at the first time step.
 - `output_writer`: Function that controls out to save the computed diagnostic
   variable to disk.
 - `reduction_time_func`: If not `nothing`, the `ScheduledDiagnostic` receives an
@@ -174,6 +175,17 @@ Given a timestep `dt`, a `ScheduledDiagnosticIterations` can be obtained from a
 `ScheduledDiagnosticTime` `sd` simply by calling
 ``ScheduledDiagnosticIterations(sd, dt)`.
 
+## The NetCDF output
+
+The NetCDF writer in `ClimaAtmos` saves different diagnostics to different files
+in the same output folder. Files are named after a combination of the diagnostic
+variable `short_name`, and the details of the temporal reduction. Inside each
+NetCDF file, there is only one diagnostic variable, along with the various
+dimensions (e.g., `lat`, `lon`, and `z`). For simulations with topography, the
+variable `z` is no longer a simple vector `z[k]`, but it is a full
+multidimensional array `z[i, j, k]` which defines the elevation on the sea level
+of the point of indices `[i, j, k]`.
+
 ## I want to add a new diagnostic variable
 
 Diagnostic variables are represented in `ClimaAtmos` with a `DiagnosticVariable`

examples/hybrid/driver.jl

@@ -5,10 +5,11 @@ Random.seed!(1234)
 if !(@isdefined config)
     config = CA.AtmosConfig()
 end
-integrator = CA.get_integrator(config)
-sol_res = CA.solve_atmos!(integrator)
+simulation = CA.get_simulation(config)
+(; integrator) = simulation
+sol_res = CA.solve_atmos!(simulation)
 
-(; simulation, atmos, params) = integrator.p
+(; atmos, params) = integrator.p
 (; p) = integrator
 
 import ClimaCore

perf/benchmark.jl

@@ -9,7 +9,8 @@ config_file = ARGS[1]
 config_dict = YAML.load_file(config_file)
 config = AtmosCoveragePerfConfig(config_dict)
 
-integrator = CA.get_integrator(config)
+simulation = CA.get_simulation(config)
+(; integrator) = simulation
 
 (; parsed_args) = config
 

perf/benchmark_dump.jl

@@ -16,7 +16,8 @@ steptimes = []
 for h_elem in 8:8:40
     config_dict["h_elem"] = h_elem
     config = CA.AtmosConfig(config_dict)
-    integrator = CA.get_integrator(config)
+    simulation = CA.get_simulation(config)
+    (; integrator) = simulation
     Y₀ = deepcopy(integrator.u)
 
     @info "Compiling benchmark_step for h_elem=$h_elem"

perf/benchmark_step.jl

@@ -18,7 +18,8 @@ import ClimaComms
 
 config = CA.AtmosConfig()
 
-integrator = CA.get_integrator(config);
+simulation = CA.get_simulation(config)
+(; integrator) = simulation;
 Y₀ = deepcopy(integrator.u);
 @info "Compiling benchmark_step!..."
 CA.benchmark_step!(integrator, Y₀); # compile first