Merge pull request #884 from CliMA/gb/benchmark_with_io

Add benchmark job with IO

.buildkite/benchmarks/pipeline.yml

@@ -82,6 +82,19 @@ steps:
           slurm_nodes: 1
           slurm_mem_per_cpu: 4GB
 
+      - label: "CPU AMIP with diagnostic EDMF and io"
+        key: "amip_diagedmf_io"
+        command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf_io.yml --job_id amip_diagedmf_io"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf_io_artifacts/*"
+        env:
+          CLIMACOMMS_CONTEXT: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+          CLIMACOMMS_DEVICE: "CPU"
+        agents:
+          slurm_ntasks_per_node: 64
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 4GB
+
   - group: "GPU benchmarks"
     steps:
       - label: "GPU ClimaAtmos without diagnostic EDMF"
@@ -123,19 +136,34 @@ steps:
           slurm_ntasks: 4
           slurm_mem: 16GB
 
+      - label: "GPU AMIP with diagnostic EDMF and IO"
+        key: "gpu_amip_diagedmf_io"
+        command: "srun julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf_io.yml --job_id gpu_amip_diagedmf_io"
+        artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf_io_artifacts/*"
+        env:
+          CLIMACOMMS_CONTEXT: "MPI"
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus_per_task: 1
+          slurm_cpus_per_task: 4
+          slurm_ntasks: 4
+          slurm_mem: 16GB
+
   - group: "Generate output table"
     steps:
       - label: "Compare AMIP/Atmos-only with diagnostic EDMF"
         key: "compare_amip_climaatmos_amip_diagedmf"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/user_io/benchmarks.jl --cpu_job_id_coupled amip_diagedmf --cpu_job_id_atmos_diagedmf climaatmos_diagedmf --cpu_job_id_atmos climaatmos --build_id $BUILDKITE_BUILD_NUMBER"
+        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/user_io/benchmarks.jl --cpu_job_id_coupled amip_diagedmf --cpu_job_id_coupled_io amip_diagedmf_io --cpu_job_id_atmos_diagedmf climaatmos_diagedmf --cpu_job_id_atmos climaatmos --build_id $BUILDKITE_BUILD_NUMBER"
         artifact_paths: "experiments/ClimaEarth/output/compare_amip_climaatmos_amip_diagedmf/*"
         depends_on:
           - "climaatmos"
           - "climaatmos_diagedmf"
           - "amip_diagedmf"
+          - "amip_diagedmf_io"
           - "gpu_climaatmos"
           - "gpu_climaatmos_diagedmf"
           - "gpu_amip_diagedmf"
+          - "gpu_amip_diagedmf_io"
 
       - label: ":envelope: Slack report: CPU/GPU AMIP/Atmos-only table"
         depends_on:

config/benchmark_configs/amip_diagedmf_io.yml

@@ -0,0 +1,16 @@
+FLOAT_TYPE: "Float32"
+anim: false
+atmos_config_file: "config/benchmark_configs/climaatmos_diagedmf_io.yml"
+atmos_config_repo: "ClimaCoupler"
+dt_cpl: 120
+dt_save_state_to_disk: "12hours"
+dt_save_to_sol: "12hours"
+energy_check: false
+land_albedo_type: "map_temporal"
+mode_name: "amip"
+mono_surface: false
+monthly_checkpoint: false
+start_date: "19790301"
+t_end: "12hours"
+turb_flux_partition: "CombinedStateFluxesMOST"
+use_coupler_diagnostics: true

config/benchmark_configs/climaatmos_diagedmf_io.yml

@@ -0,0 +1,34 @@
+FLOAT_TYPE: "Float32"
+approximate_linear_solve_iters: 2
+dt: 120secs
+dt_cloud_fraction: 1hours
+dt_rad: 1hours
+dt_save_state_to_disk: "12hours"
+dt_save_to_sol: "12hours"
+dz_bottom: 30.0
+dz_top: 3000.0
+edmfx_detr_model: "Generalized"
+edmfx_entr_model: "Generalized"
+edmfx_nh_pressure: true
+edmfx_sgs_diffusive_flux: true
+edmfx_sgs_mass_flux: true
+edmfx_upwinding: first_order
+h_elem: 30
+idealized_insolation: false
+implicit_diffusion: true
+moist: equil
+ode_algo: ARS343
+precip_model: 0M
+prognostic_tke: true
+rad: allskywithclear
+surface_setup: DefaultMoninObukhov
+t_end: 12hours
+turb_flux_partition: "CombinedStateFluxesMOST"
+toml: [toml/diagnostic_edmfx.toml]
+turbconv: diagnostic_edmfx
+z_elem: 63
+z_max: 55000.0
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [pfull, rsut, rlut, pr, hus, rv]
+    period: 10hours

experiments/ClimaEarth/user_io/benchmarks.jl

@@ -36,6 +36,14 @@ function argparse_settings()
         help = "The name of the GPU coupled run we want to compare."
         arg_type = String
         default = nothing
+        "--cpu_job_id_coupled_io"
+        help = "The name of the CPU coupled run with IO we want to compare. User must specify CPU and/or GPU coupled run name."
+        arg_type = String
+        default = nothing
+        "--gpu_job_id_coupled_io"
+        help = "The name of the GPU coupled with IO run we want to compare."
+        arg_type = String
+        default = nothing
         "--cpu_job_id_atmos"
         help = "The name of the CPU atmos-only run without diagnostic EDMF we want to compare. User must specify CPU and/or GPU atmos-only non-EDMF run name."
         arg_type = String
@@ -70,14 +78,18 @@ end
 Use the input `parsed_args` to get the job ID and artifacts directories for
 both the CPU and GPU runs of the given `run_type`.
 
-`run_type` must be one of "coupled", "atmos", or "atmos_diagedmf".
+`run_type` must be one of "coupled", "coupled_io", "atmos", or "atmos_diagedmf".
 """
 function get_run_info(parsed_args, run_type)
     # Read in CPU and GPU job ID info from command line
     if run_type == "coupled"
         cpu_job_id = parsed_args["cpu_job_id_coupled"]
         gpu_job_id = parsed_args["gpu_job_id_coupled"]
         mode_name = "amip"
+    elseif run_type == "coupled_io"
+        cpu_job_id = parsed_args["cpu_job_id_coupled_io"]
+        gpu_job_id = parsed_args["gpu_job_id_coupled_io"]
+        mode_name = "amip"
     elseif run_type == "atmos_diagedmf"
         cpu_job_id = parsed_args["cpu_job_id_atmos_diagedmf"]
         gpu_job_id = parsed_args["gpu_job_id_atmos_diagedmf"]
@@ -160,6 +172,7 @@ end
 
 # Read in run info for each of the cases we want to compare
 run_info_coupled = get_run_info(parsed_args, "coupled")
+run_info_coupled_io = get_run_info(parsed_args, "coupled_io")
 run_info_atmos_diagedmf = get_run_info(parsed_args, "atmos_diagedmf")
 run_info_atmos = get_run_info(parsed_args, "atmos")
 
@@ -172,6 +185,7 @@ data = [
 
 # Append data to the table for each of the cases we want to compare
 data = append_table_data(data, "Coupled", run_info_coupled...)
+data = append_table_data(data, "Coupled with IO", run_info_coupled_io...)
 data = append_table_data(data, "Atmos with diag. EDMF", run_info_atmos_diagedmf...)
 data = append_table_data(data, "Atmos without diag. EDMF", run_info_atmos...)
 

test/runtests.jl

@@ -56,9 +56,6 @@ end
 @safetestset "experiment test: Leaderboard" begin
     include("experiment_tests/leaderboard.jl")
 end
-@safetestset "component test: bucket" begin
-    include("component_model_tests/bucket_tests.jl")
-end
 @safetestset "component model test: ClimaAtmos" begin
     include("component_model_tests/climaatmos_tests.jl")
 end