Merge remote-tracking branch 'origin/main' into as/covar-deriv

CliMA · Sep 30, 2024 · 1871f37 · 1871f37
2 parents b727eb4 + db918c7
commit 1871f37
Show file tree

Hide file tree

Showing 78 changed files with 843 additions and 252 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -56,11 +56,11 @@ steps:
 
   - wait
 
-  - group: "Regression tests"
+  - group: "Reproducibility tests"
     steps:
 
       - label: ":computer: Ensure mse tables are reset when necessary"
-        command: "julia --color=yes --project=examples regression_tests/test_reset.jl"
+        command: "julia --color=yes --project=examples reproducibility_tests/test_reset.jl"
 
   - group: "Radiation"
     steps:
@@ -256,7 +256,7 @@ steps:
           --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist.yml
           --job_id sphere_baroclinic_wave_rhoe_equilmoist
 
-          julia --color=yes --project=examples regression_tests/test_mse.jl
+          julia --color=yes --project=examples reproducibility_tests/test_mse.jl
           --job_id sphere_baroclinic_wave_rhoe_equilmoist
           --out_dir sphere_baroclinic_wave_rhoe_equilmoist/output_active
         artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist/output_active/*"
@@ -269,7 +269,7 @@ steps:
           --config_file $CONFIG_PATH/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml
           --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist
 
-          julia --color=yes --project=examples regression_tests/test_mse.jl
+          julia --color=yes --project=examples reproducibility_tests/test_mse.jl
           --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist
           --out_dir deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active
         artifact_paths: "deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active/*"
@@ -302,7 +302,7 @@ steps:
           --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml
           --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge
 
-          julia --color=yes --project=examples regression_tests/test_mse.jl
+          julia --color=yes --project=examples reproducibility_tests/test_mse.jl
           --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge
           --out_dir sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active
         artifact_paths: "sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active/*"
@@ -327,7 +327,7 @@ steps:
           --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml
           --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric
 
-          julia --color=yes --project=examples regression_tests/test_mse.jl
+          julia --color=yes --project=examples reproducibility_tests/test_mse.jl
           --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric
           --out_dir sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active
         artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active/*"
@@ -411,6 +411,44 @@ steps:
           --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth
         artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth/output_active/*"
 
+  - group: "Restarting"
+    steps:
+
+      - label: ":computer: test restart"
+        command: >
+          julia --color=yes --project=examples test/restart.jl
+        agents:
+          slurm_mem: 16GB
+
+      - label: ":computer: test restart GPU"
+        command: >
+          julia --color=yes --project=examples test/restart.jl
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+          slurm_mem: 16G
+
+      # - label: ":computer: test restart MPI"
+      #   command: >
+      #     srun julia --color=yes --project=examples test/restart.jl
+      #   env:
+      #     CLIMACOMMS_CONTEXT: "MPI"
+      #   agents:
+      #     slurm_ntasks: 2
+      #     slurm_mem: 16G
+
+      # - label: ":computer: test restart GPU MPI"
+      #   command: >
+      #     srun julia --color=yes --project=examples test/restart.jl
+      #   env:
+      #     CLIMACOMMS_CONTEXT: "MPI"
+      #     CLIMACOMMS_DEVICE: "CUDA"
+      #   agents:
+      #     slurm_gpus_per_task: 1
+      #     slurm_ntasks: 2
+      #     slurm_mem: 16G
+
   - group: "MPI Examples"
     steps:
 
@@ -610,7 +648,7 @@ steps:
           --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml
           --job_id diagnostic_edmfx_aquaplanet
 
-          julia --color=yes --project=examples regression_tests/test_mse.jl
+          julia --color=yes --project=examples reproducibility_tests/test_mse.jl
           --job_id diagnostic_edmfx_aquaplanet
           --out_dir diagnostic_edmfx_aquaplanet/output_active
         artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*"
@@ -1111,10 +1149,10 @@ steps:
     continue_on_failure: true
 
   - label: ":robot_face: Print new mse tables"
-    command: "julia --color=yes --project=examples regression_tests/print_new_mse.jl"
+    command: "julia --color=yes --project=examples reproducibility_tests/print_new_mse.jl"
 
   - label: ":robot_face: Print new reference counter"
-    command: "julia --color=yes --project=examples regression_tests/print_new_ref_counter.jl"
+    command: "julia --color=yes --project=examples reproducibility_tests/print_new_ref_counter.jl"
 
   - label: ":bar_chart: Tabulate performance summary"
     command: "julia --color=yes --project=perf perf/tabulate_perf_summary.jl"
@@ -1128,4 +1166,4 @@ steps:
   - wait
 
   - label: ":robot_face: Move main results"
-    command: "julia --color=yes --project=examples regression_tests/move_output.jl"
+    command: "julia --color=yes --project=examples reproducibility_tests/move_output.jl"
diff --git a/config/default_configs/default_config.yml b/config/default_configs/default_config.yml
@@ -218,8 +218,8 @@ non_orographic_gravity_wave:
 nh_poly:
   help: "Horizontal polynomial degree. Note: The number of quadrature points in 1D within each horizontal element is then Nq = <--nh_poly> + 1"
   value: 3
-regression_test:
-  help: "(Bool) perform regression test"
+reproducibility_test:
+  help: "(Bool) perform reproducibility test"
   value: false
 check_conservation:
   help: "Check conservation of mass and energy [`false` (default), `true`]"
@@ -287,6 +287,9 @@ restart_file:
 prescribe_ozone:
   help: "Prescribe time and spatially varying ozone from a file [`false` (default), `true`]"
   value: false
+detect_restart_file:
+  help: "When true, try finding a restart file and use it to restart the simulation. Only works with ActiveLink."
+  value: false
 prescribed_aerosols:
   help: "Which aerosols to add. List of keys from the data file (e.g., CB1, CB2)."
   value: []

diff --git a/config/gpu_configs/gpu_aquaplanet_dyamond_diag_1process.yml b/config/gpu_configs/gpu_aquaplanet_dyamond_diag_1process.yml
@@ -1,9 +1,11 @@
 dt_save_state_to_disk: 12hours
 dt_save_to_sol: 12hours
 h_elem: 30
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
+rayleigh_sponge: true
+viscous_sponge: true
 moist: equil
 precip_model: 1M
 rad: allskywithclear
@@ -14,7 +16,6 @@ implicit_diffusion: true
 approximate_linear_solve_iters: 2
 dt_cloud_fraction: 1hours
 surface_setup: DefaultMoninObukhov
-rayleigh_sponge: true
 dt: 90secs
 t_end: 1days
 toml: [toml/longrun_aquaplanet.toml]

diff --git a/config/gpu_configs/gpu_aquaplanet_dyamond_ss.yml b/config/gpu_configs/gpu_aquaplanet_dyamond_ss.yml
@@ -2,9 +2,11 @@ dt_save_state_to_disk: "Inf"
 dt_save_to_sol: "Inf"
 output_default_diagnostics: false
 h_elem: 30
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
+rayleigh_sponge: true
+viscous_sponge: true
 moist: "equil"
 precip_model: "1M"
 rad: "allskywithclear"
@@ -15,7 +17,6 @@ vert_diff: "FriersonDiffusion"
 implicit_diffusion: true
 approximate_linear_solve_iters: 2
 surface_setup: "DefaultMoninObukhov"
-rayleigh_sponge: true
 dt: "90secs"
 t_end: "1days"
 toml: [toml/longrun_aquaplanet.toml]

diff --git a/config/gpu_configs/gpu_aquaplanet_dyamond_ws_1process.yml b/config/gpu_configs/gpu_aquaplanet_dyamond_ws_1process.yml
@@ -2,9 +2,11 @@ dt_save_state_to_disk: "Inf"
 dt_save_to_sol: "Inf"
 output_default_diagnostics: false
 h_elem: 30
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
+rayleigh_sponge: true
+viscous_sponge: true
 moist: "equil"
 precip_model: "1M"
 rad: "allskywithclear"

diff --git a/config/gpu_configs/gpu_aquaplanet_dyamond_ws_2process.yml b/config/gpu_configs/gpu_aquaplanet_dyamond_ws_2process.yml
@@ -2,9 +2,11 @@ dt_save_state_to_disk: "Inf"
 dt_save_to_sol: "Inf"
 output_default_diagnostics: false
 h_elem: 42
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
+rayleigh_sponge: true
+viscous_sponge: true
 moist: "equil"
 precip_model: "1M"
 rad: "allskywithclear"

diff --git a/config/gpu_configs/gpu_aquaplanet_dyamond_ws_4process.yml b/config/gpu_configs/gpu_aquaplanet_dyamond_ws_4process.yml
@@ -2,9 +2,11 @@ dt_save_state_to_disk: "Inf"
 dt_save_to_sol: "Inf"
 output_default_diagnostics: false
 h_elem: 60
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
+rayleigh_sponge: true
+viscous_sponge: true
 moist: "equil"
 precip_model: "1M"
 rad: "allskywithclear"
@@ -15,7 +17,6 @@ vert_diff: "FriersonDiffusion"
 implicit_diffusion: true
 approximate_linear_solve_iters: 2
 surface_setup: "DefaultMoninObukhov"
-rayleigh_sponge: true
 dt: "90secs"
 t_end: "1days"
 toml: [toml/longrun_aquaplanet.toml]

diff --git a/config/gpu_configs/gpu_hs_rhoe_equil_55km_nz63_0M.yml b/config/gpu_configs/gpu_hs_rhoe_equil_55km_nz63_0M.yml
@@ -12,4 +12,4 @@ moist: "equil"
 precip_model: "0M"
 rayleigh_sponge: true
 forcing: "held_suarez"
-toml: [toml/longrun_aquaplanet.toml]
+toml: [toml/longrun_held_suarez.toml]
diff --git a/config/longrun_configs/amip_target_diagedmf.yml b/config/longrun_configs/amip_target_diagedmf.yml
@@ -1,9 +1,11 @@
-dt_save_to_sol: "10days"
-dt_save_state_to_disk: "20days"
 h_elem: 16
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
+rayleigh_sponge: true
+viscous_sponge: true
+dt_save_to_sol: "10days"
+dt_save_state_to_disk: "20days"
 moist: "equil"
 cloud_model: "quadrature_sgs"
 precip_model: "0M"
@@ -27,7 +29,6 @@ edmfx_detr_model: "Generalized"
 edmfx_nh_pressure: true
 edmfx_sgs_mass_flux: true
 edmfx_sgs_diffusive_flux: true
-rayleigh_sponge: true
 dt: "120secs"
 t_end: "120days"
 toml: [toml/longrun_aquaplanet_diagedmf.toml]

diff --git a/config/longrun_configs/longrun_aquaplanet_allsky_0M.yml b/config/longrun_configs/longrun_aquaplanet_allsky_0M.yml
@@ -1,8 +1,9 @@
 h_elem: 16
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
 rayleigh_sponge: true
+viscous_sponge: true
 implicit_diffusion: true
 approximate_linear_solve_iters: 2
 dt: "120secs"

diff --git a/config/longrun_configs/longrun_aquaplanet_allsky_0M_earth.yml b/config/longrun_configs/longrun_aquaplanet_allsky_0M_earth.yml
@@ -1,8 +1,9 @@
 h_elem: 16
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
 rayleigh_sponge: true
+viscous_sponge: true
 topography: "Earth"
 topo_smoothing: true
 implicit_diffusion: true

diff --git a/config/longrun_configs/longrun_aquaplanet_allsky_1M.yml b/config/longrun_configs/longrun_aquaplanet_allsky_1M.yml
@@ -1,8 +1,9 @@
 h_elem: 16
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
 rayleigh_sponge: true
+viscous_sponge: true
 implicit_diffusion: true
 approximate_linear_solve_iters: 2
 dt: "120secs"

diff --git a/config/longrun_configs/longrun_aquaplanet_allsky_diagedmf_0M.yml b/config/longrun_configs/longrun_aquaplanet_allsky_diagedmf_0M.yml
@@ -1,8 +1,9 @@
 h_elem: 16
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
 rayleigh_sponge: true
+viscous_sponge: true
 implicit_diffusion: true
 approximate_linear_solve_iters: 2
 ode_algo: ARS343

diff --git a/config/longrun_configs/longrun_aquaplanet_allsky_progedmf_diffonly_0M.yml b/config/longrun_configs/longrun_aquaplanet_allsky_progedmf_diffonly_0M.yml
@@ -1,8 +1,9 @@
 h_elem: 16
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
 rayleigh_sponge: true
+viscous_sponge: true
 implicit_diffusion: true
 approximate_linear_solve_iters: 2
 max_newton_iters_ode: 3

diff --git a/config/longrun_configs/longrun_aquaplanet_allsky_tvinsol_0M_slabocean.yml b/config/longrun_configs/longrun_aquaplanet_allsky_tvinsol_0M_slabocean.yml
@@ -1,8 +1,9 @@
 h_elem: 16
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
 rayleigh_sponge: true
+viscous_sponge: true
 implicit_diffusion: true
 approximate_linear_solve_iters: 2
 dt: "120secs"

diff --git a/config/longrun_configs/longrun_aquaplanet_dyamond.yml b/config/longrun_configs/longrun_aquaplanet_dyamond.yml
@@ -1,8 +1,9 @@
 h_elem: 60
-z_max: 55000.0
+z_max: 65000.0
 z_elem: 63
 dz_bottom: 30.0
 rayleigh_sponge: true
+viscous_sponge: true
 moist: "equil" 
 cloud_model: "grid_scale"
 dt: "30secs" 

diff --git a/config/longrun_configs/longrun_dry_held_suarez.yml b/config/longrun_configs/longrun_dry_held_suarez.yml
@@ -1,10 +1,10 @@
 h_elem: 30
-z_elem: 63
 z_max: 55000.0
+z_elem: 63
 dz_bottom: 30.0
 dt: "90secs"
 t_end: "360days"
 forcing: "held_suarez"
 rayleigh_sponge: true
 dt_save_state_to_disk: "10days"
-toml: [toml/longrun_aquaplanet.toml]
+toml: [toml/longrun_held_suarez.toml]
diff --git a/config/longrun_configs/longrun_moist_held_suarez.yml b/config/longrun_configs/longrun_moist_held_suarez.yml
@@ -1,6 +1,6 @@
 h_elem: 30
-z_elem: 63
 z_max: 55000.0
+z_elem: 63
 dz_bottom: 30.0
 dt: "90secs"
 t_end: "360days"
@@ -10,4 +10,4 @@ vert_diff: true
 rayleigh_sponge: true
 forcing: "held_suarez"
 dt_save_state_to_disk: "10days"
-toml: [toml/longrun_aquaplanet.toml]
+toml: [toml/longrun_held_suarez.toml]
diff --git a/config/longrun_configs/longrun_moist_held_suarez_deepatmos.yml b/config/longrun_configs/longrun_moist_held_suarez_deepatmos.yml
@@ -1,6 +1,6 @@
 h_elem: 30
-z_elem: 63
 z_max: 55000.0
+z_elem: 63
 dz_bottom: 30.0
 dt: "90secs"
 t_end: "360days"
@@ -11,4 +11,4 @@ vert_diff: true
 rayleigh_sponge: true
 forcing: "held_suarez"
 dt_save_state_to_disk: "10days"
-toml: [toml/longrun_aquaplanet.toml]
+toml: [toml/longrun_held_suarez.toml]
diff --git a/config/longrun_configs/longrun_sphere_hydrostatic_balance_rhoe.yml b/config/longrun_configs/longrun_sphere_hydrostatic_balance_rhoe.yml
@@ -1,10 +1,9 @@
-dt_save_state_to_disk: "10days"
-dt: "90secs"
-t_end: "360days"
 h_elem: 30
+z_max: 55000.0
 z_elem: 63
 dz_bottom: 30.0
-z_max: 55000.0
+dt_save_state_to_disk: "10days"
+dt: "90secs"
+t_end: "360days"
 perturb_initstate: false
 discrete_hydrostatic_balance: true
-toml: [toml/longrun_aquaplanet.toml]