From b6f9ebe76f2dda22651af21e486e2da4d4ca251a Mon Sep 17 00:00:00 2001 From: Zhaoyi Shen <11598433+szy21@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:43:17 -0700 Subject: [PATCH] wip --- .buildkite/longruns/pipeline.yml | 40 +- .buildkite/pipeline.yml | 1652 ++++++++--------- ...lanet_equil_helem12_allsky_progedmf_0M.yml | 38 + ...n_aquaplanet_equil_helem12_clearsky_0M.yml | 23 + ...anet_equil_helem12_clearsky_0M_viscous.yml | 24 + ...net_equil_helem12_clearsky_progedmf_0M.yml | 39 + ...quil_helem12_clearsky_progedmf_0M_asym.yml | 38 + ...e_equil_55km_nz63_clearsky_progedmf_0M.yml | 37 + .../prognostic_edmfx_adv_test_column.yml | 4 +- .../prognostic_edmfx_aquaplanet.yml | 7 +- .../prognostic_edmfx_trmm_column_0M.yml | 4 +- post_processing/ci_plots.jl | 19 +- src/initial_conditions/initial_conditions.jl | 9 +- src/prognostic_equations/edmfx_closures.jl | 48 + src/prognostic_equations/edmfx_sgs_flux.jl | 35 + src/prognostic_equations/edmfx_tke.jl | 4 +- src/prognostic_equations/hyperdiffusion.jl | 2 +- ..._aquaplanet_equil_helem12_clearsky_0M.toml | 11 + ...net_equil_helem12_clearsky_0M_viscous.toml | 14 + ...et_equil_helem12_clearsky_progedmf_0M.toml | 50 + ..._equil_55km_nz63_clearsky_progedmf_0M.toml | 2 +- 21 files changed, 1251 insertions(+), 849 deletions(-) create mode 100644 config/longrun_configs/longrun_aquaplanet_equil_helem12_allsky_progedmf_0M.yml create mode 100644 config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M.yml create mode 100644 config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.yml create mode 100644 config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.yml create mode 100644 config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M_asym.yml create mode 100644 config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.yml create mode 100644 toml/longrun_aquaplanet_equil_helem12_clearsky_0M.toml create mode 100644 toml/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.toml create mode 100644 toml/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.toml diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml index bd5396a1657..05d7a540ccc 100644 --- a/.buildkite/longruns/pipeline.yml +++ b/.buildkite/longruns/pipeline.yml @@ -1,6 +1,5 @@ agents: queue: new-central - slurm_mem_per_cpu: 8G modules: climacommon/2024_04_30 env: @@ -33,6 +32,7 @@ steps: agents: slurm_cpus_per_task: 8 slurm_gpus: 1 + slurm_ntasks: 1 env: JULIA_NUM_PRECOMPILE_TASKS: 8 @@ -54,6 +54,7 @@ steps: # env: # JOB_NAME: "longrun_zalesak_tracer_energy_bw_rhoe_equil_highres" +<<<<<<< HEAD - label: ":computer: SSP baroclinic wave (ρe_tot) equilmoist high resolution centered diff" command: - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME @@ -64,22 +65,45 @@ steps: slurm_time: 24:00:00 env: JOB_NAME: "longrun_ssp_bw_rhoe_equil_highres" +======= + # - label: ":computer: SSP baroclinic wave (ρe_tot) equilmoist high resolution centered diff" + # command: + # - "srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml" + # artifact_paths: "$$JOB_NAME/output_active/*" + # agents: + # slurm_ntasks: 32 + # slurm_nodes: 2 + # slurm_time: 24:00:00 + # env: + # JOB_NAME: "longrun_ssp_bw_rhoe_equil_highres" +>>>>>>> fec1acee0 (wip) + + # - label: ":computer: aquaplanet equilmoist clearsky radiation + prognostic edmf diffusion only + 0M microphysics" + # command: + # - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml + # artifact_paths: "$$JOB_NAME/output_active/*" + # agents: + # slurm_ntasks: 64 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16GB + # slurm_time: 24:00:00 + # env: + # JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_diffonly_0M" - - label: ":computer: aquaplanet equilmoist clearsky radiation + prognostic edmf diffusion only + 0M microphysics" + - label: ":computer: aquaplanet equilmoist allsky radiation + prognostic edmf + 0M microphysics" command: - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: - slurm_ntasks: 64 - slurm_nodes: 4 - slurm_mem_per_cpu: 16GB + slurm_gpus: 1 slurm_time: 24:00:00 + slurm_mem: 32GB env: - JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_diffonly_0M" + JOB_NAME: "longrun_aquaplanet_equil_helem12_allsky_progedmf_0M" - - group: "Low resolution long runs" + # - group: "Low resolution long runs" - steps: + # steps: - label: ":computer: low resolution aquaplanet equilmoist clearsky radiation + time-varying insolation + slab ocean" command: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d7b62bd0b67..240ff4f7d63 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -51,552 +51,552 @@ steps: - wait - - group: "Regression tests" - steps: - - - label: ":computer: Ensure mse tables are reset when necessary" - command: "julia --color=yes --project=examples regression_tests/test_reset.jl" - - - group: "Radiation" - steps: - - - label: ":computer: single column radiative equilibrium gray" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_gray.yml - --job_id single_column_radiative_equilibrium_gray - artifact_paths: "single_column_radiative_equilibrium_gray/output_active/*" - - - label: ":computer: single column radiative equilibrium clearsky" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky.yml - --job_id single_column_radiative_equilibrium_clearsky - artifact_paths: "single_column_radiative_equilibrium_clearsky/output_active/*" - - - label: ":computer: single column radiative equilibrium clearsky prognostic surface temperature" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky_prognostic_surface_temp.yml - --job_id single_column_radiative_equilibrium_clearsky_prognostic_surface_temp - artifact_paths: "single_column_radiative_equilibrium_clearsky_prognostic_surface_temp/output_active/*" - - - label: ":computer: single column radiative equilibrium allsky idealized clouds varying insolation" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_allsky_idealized_clouds.yml - --job_id single_column_radiative_equilibrium_allsky_idealized_clouds - artifact_paths: "single_column_radiative_equilibrium_allsky_idealized_clouds/output_active/*" - - - group: "Precipitation" - steps: - - label: ":umbrella: 1-moment precipitation sanity test single column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_precipitation_test.yml - --job_id single_column_precipitation_test - artifact_paths: "single_column_precipitation_test/output_active/*" - - - group: "Gravity wave" - steps: - - - label: ":computer: non-orographic gravity wave parameterization unit test 3d" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_3d.jl" - artifact_paths: "nonorographic_gravity_wave_test_3d/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: non-orographic gravity wave parameterization test with MiMA output" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_mima.jl" - artifact_paths: "nonorographic_gravity_wave_test_mima/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: non-orographic gravity wave parameterization unit test single column" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_single_column.jl" - artifact_paths: "nonorographic_gravity_wave_test_single_column/output_active/*" - - - label: ":computer: orographic gravity wave parameterization unit test for base flux calculation" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_baseflux.jl" - artifact_paths: "orographic_gravity_wave_test_baseflux/output_active/*" - - - label: ":computer: orographic gravity wave parameterization unit test for 3d calculation" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_3d.jl" - artifact_paths: "orographic_gravity_wave_test_3d/output_active/*" - - - label: ":computer: single column non-orographic gravity wave parameterization" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_nonorographic_gravity_wave.yml - --job_id single_column_nonorographic_gravity_wave - artifact_paths: "single_column_nonorographic_gravity_wave/output_active/*" - - - group: "Column Examples" - steps: - - - label: ":computer: single column hydrostatic balance float64" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_hydrostatic_balance_ft64.yml - --job_id single_column_hydrostatic_balance_ft64 - artifact_paths: "single_column_hydrostatic_balance_ft64/output_active/*" - - - group: "Box Examples" - steps: - - - label: ":computer: Box hydrostatic balance (ρe_tot)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/box_hydrostatic_balance_rhoe.yml - --job_id box_hydrostatic_balance_rhoe - artifact_paths: "box_hydrostatic_balance_rhoe/output_active/*" - - - label: ":computer: 3D density current" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/box_density_current_test.yml - --job_id box_density_current_test - artifact_paths: "box_density_current_test/output_active/*" - - - group: "Plane Examples" - steps: - - label: ":computer: Agnesi linear hydrostatic mountain experiment (uniform)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_agnesi_mountain_test_uniform.yml - --job_id plane_agnesi_mountain_test_uniform - artifact_paths: "plane_agnesi_mountain_test_uniform/output_active/*" - - - label: ":computer: Agnesi linear hydrostatic mountain experiment (stretched)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_agnesi_mountain_test_stretched.yml - --job_id plane_agnesi_mountain_test_stretched - artifact_paths: "plane_agnesi_mountain_test_stretched/output_active/*" - - - label: ":computer: Schar mountain experiment (uniform grid)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_schar_mountain_test_uniform.yml - --job_id plane_schar_mountain_test_uniform - artifact_paths: "plane_schar_mountain_test_uniform/output_active/*" - - - label: ":computer: Schar mountain experiment (stretched grid)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_schar_mountain_test_stretched.yml - --job_id plane_schar_mountain_test_stretched - artifact_paths: "plane_schar_mountain_test_stretched/output_active/*" - - - label: ":computer: Density current experiment" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_density_current_test.yml - --job_id plane_density_current_test - artifact_paths: "plane_density_current_test/output_active/*" - - - - group: "Sphere Examples (Dycore)" - steps: - - - label: ":computer: hydrostatic balance (ρe) float64" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_hydrostatic_balance_rhoe_ft64.yml - --job_id sphere_hydrostatic_balance_rhoe_ft64 - artifact_paths: "sphere_hydrostatic_balance_rhoe_ft64/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: baroclinic wave (ρe)" - key: sphere_baroclinic_wave_rhoe - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe.yml - --job_id sphere_baroclinic_wave_rhoe - artifact_paths: "sphere_baroclinic_wave_rhoe/output_active/*" - - - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist.yml - --job_id sphere_baroclinic_wave_rhoe_equilmoist - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id sphere_baroclinic_wave_rhoe_equilmoist - --out_dir sphere_baroclinic_wave_rhoe_equilmoist/output_active - artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" - agents: - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist (deep sphere)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml - --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist - --out_dir deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active - artifact_paths: "deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" - agents: - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist with cloud diagnostics" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist_cldiag.yml - --job_id sphere_baroclinic_wave_rhoe_equilmoist_cldiag - artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist_cldiag/output_active/*" - - - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist explicit vertdiff" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist_expvdiff.yml - --job_id sphere_baroclinic_wave_rhoe_equilmoist_expvdiff - artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist_expvdiff/output_active/*" - - - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist implicit vertdiff" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist_impvdiff.yml - --job_id sphere_baroclinic_wave_rhoe_equilmoist_impvdiff - artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist_impvdiff/output_active/*" - - # Add this back when we figure out what to do with SSP and zalesak - # - label: ":computer: SSP zalesak tracer & energy upwind baroclinic wave (ρe_tot) equilmoist" - # command: > - # julia --color=yes --project=examples examples/hybrid/driver.jl - # --config_file $CONFIG_PATH/$$JOB_NAME.yml - # --job_id JOB_NAME - # artifact_paths: "$$JOB_NAME/output_active/*" - # agents: - # slurm_mem: 64GB - # env: - # JOB_NAME: "sphere_zalesak_upwind_tracer_energy_ssp_baroclinic_wave_rhoe_equilmoist" - - - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist check conservation float64" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist_conservation_ft64.yml - --job_id sphere_baroclinic_wave_rhoe_equilmoist_conservation_ft64 - artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist_conservation_ft64/output_active/*" - - - label: ":computer: held suarez (ρe) hightop" - key: sphere_held_suarez_rhoe_hightop - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_hightop.yml - --job_id sphere_held_suarez_rhoe_hightop - artifact_paths: "sphere_held_suarez_rhoe_hightop/output_active/*" - - - label: ":computer: no lim ARS held suarez (ρe) equilmoist hightop sponge" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml - --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge - --out_dir sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active - artifact_paths: "sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active/*" - agents: - slurm_constraint: icelake|cascadelake|skylake|epyc - - - group: "Sphere Examples (Aquaplanet)" - steps: - - - label: ":umbrella: aquaplanet (ρe_tot) equilmoist allsky radiation monin_obukhov varying insolation gravity wave (gfdl_restart) high top with 1-moment micro" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res.yml - --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res - artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: aquaplanet (ρe_tot) nonequilmoist allsky radiation monin_obukhov varying insolation gravity wave (gfdl_restart) high top with 1-moment micro" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_nonequilmoist_allsky_gw_res.yml - --job_id sphere_aquaplanet_rhoe_nonequilmoist_allsky_gw_res - artifact_paths: "sphere_aquaplanet_rhoe_nonequilmoist_allsky_gw_res/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: aquaplanet (ρe_tot) equilmoist allsky radiation monin_obukhov varying insolation gravity wave (raw_topo) high top zonally asymmetric" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml - --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric - --out_dir sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active - artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":computer: aquaplanet (ρe_tot) slabocean equilmoist allsky radiation monin_obukhov varying insolation" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean.yml - --job_id aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean - artifact_paths: "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean/output_active/*" - agents: - slurm_mem: 20GB - - - group: "Sphere Examples (Topography)" - steps: - - - label: ":computer: baroclinic wave (ρe) topography (dcmip)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_topography_dcmip_rs.yml - --job_id sphere_baroclinic_wave_rhoe_topography_dcmip_rs - artifact_paths: "sphere_baroclinic_wave_rhoe_topography_dcmip_rs/output_active/*" - - - label: ":computer: held suarez (ρe) topography (dcmip)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_topography_dcmip.yml - --job_id sphere_held_suarez_rhoe_topography_dcmip - artifact_paths: "sphere_held_suarez_rhoe_topography_dcmip/output_active/*" - - - label: ":computer: held suarez (ρe) equilmoist topography (dcmip)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_topography_dcmip.yml - --job_id sphere_held_suarez_rhoe_equilmoist_topography_dcmip - artifact_paths: "sphere_held_suarez_rhoe_equilmoist_topography_dcmip/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: Diagnostic DCMIP200 surface elevation spectra" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200.yml - --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200 - artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200/output_active/*" - - - label: ":computer: Diagnostic Earth surface elevation spectra" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth.yml - --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth - artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth/output_active/*" - - - group: "MPI Examples" - steps: - - - label: ":computer: Prep restart for MPI" - key: "mpi_baro_wave_make_restart" - command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $MPI_CONFIG_PATH/mpi_make_restart.yml - --job_id mpi_make_restart - artifact_paths: "mpi_make_restart/output_active/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - - - label: ":computer: Test restart for MPI baroclinic wave" - key: "restart_mpi_baro_wave" - depends_on: "mpi_baro_wave_make_restart" - command: > - tar xvf mpi_make_restart/output_active/hdf5_files.tar -C mpi_make_restart/output_active - - srun julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $MPI_CONFIG_PATH/restart_mpi_baroclinic_wave_rhoe.yml - --job_id restart_mpi_baroclinic_wave_rhoe - artifact_paths: "restart_mpi_baroclinic_wave_rhoe/output_active/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - timeout_in_minutes: 20 - soft_fail: true - #retry: - # automatic: true - - - label: ":computer: MPI no lim ARS aquaplanet (ρe) equilmoist clearsky radiation" - command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $MPI_CONFIG_PATH/mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky.yml - --job_id mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky - artifact_paths: "mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky/output_active/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: ":computer: Prep for calling remap pipeline" - key: "prep_remap" - command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $MPI_CONFIG_PATH/prep_remap.yml - --job_id prep_remap - artifact_paths: "prep_remap/output_active/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - - - label: ":computer: Exercise remap pipeline" - key: "remap_pipeline" - depends_on: "prep_remap" - command: > - tar xvf prep_remap/output_active/hdf5_files.tar -C prep_remap/output_active - - julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl - --data_dir prep_remap/output_active --out_dir remap_pipeline_output - artifact_paths: "remap_pipeline_output/*" - - - group: "Configs" - steps: - - - label: ":computer: Test IO" - command: > - julia --color=yes --project=examples --threads=8 examples/hybrid/driver.jl - --config_file $CONFIG_PATH/test_io.yml - --job_id test_io - artifact_paths: "test_io/output_active/*" - agents: - slurm_cpus_per_task: 8 - - - label: ":computer: MPI io test" - command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/test_mpi_io.yml - --job_id test_mpi_io - artifact_paths: "test_mpi_io/output_active/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - - # TODO: we should somehow decouple this unit test from the perf env / scripts - - label: ":computer: checkbounds" - command: > - julia --color=yes --check-bounds=yes --project=perf perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/checkbounds.yml - --job_id checkbounds - artifact_paths: "checkbounds/output_active/*" - agents: - slurm_mem: 20GB - - - group: "Diagnostic EDMFX" - steps: - - - label: ":genie: Diagnostic EDMFX test in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_test_box.yml - --job_id diagnostic_edmfx_test_box - artifact_paths: "diagnostic_edmfx_test_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX GABLS in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_gabls_box.yml - --job_id diagnostic_edmfx_gabls_box - artifact_paths: "diagnostic_edmfx_gabls_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX Bomex in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_box.yml - --job_id diagnostic_edmfx_bomex_box - artifact_paths: "diagnostic_edmfx_bomex_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX Bomex stretched grid in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_stretched_box.yml - --job_id diagnostic_edmfx_bomex_stretched_box - artifact_paths: "diagnostic_edmfx_bomex_stretched_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box (explicit)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_explicit_box.yml - --job_id diagnostic_edmfx_dycoms_rf01_explicit_box - artifact_paths: "diagnostic_edmfx_dycoms_rf01_explicit_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_box.yml - --job_id diagnostic_edmfx_dycoms_rf01_box - artifact_paths: "diagnostic_edmfx_dycoms_rf01_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX Rico in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_rico_box.yml - --job_id diagnostic_edmfx_rico_box - artifact_paths: "diagnostic_edmfx_rico_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX TRMM in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box.yml - --job_id diagnostic_edmfx_trmm_box - artifact_paths: "diagnostic_edmfx_trmm_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX TRMM stretched grid in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_stretched_box.yml - --job_id diagnostic_edmfx_trmm_stretched_box - artifact_paths: "diagnostic_edmfx_trmm_stretched_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX TRMM 0M in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box_0M.yml - --job_id diagnostic_edmfx_trmm_box_0M - artifact_paths: "diagnostic_edmfx_trmm_box_0M/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX aquaplanet with TKE" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml - --job_id diagnostic_edmfx_aquaplanet - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id diagnostic_edmfx_aquaplanet - --out_dir diagnostic_edmfx_aquaplanet/output_active - artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc + # - group: "Regression tests" + # steps: + + # - label: ":computer: Ensure mse tables are reset when necessary" + # command: "julia --color=yes --project=examples regression_tests/test_reset.jl" + + # - group: "Radiation" + # steps: + + # - label: ":computer: single column radiative equilibrium gray" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_gray.yml + # --job_id single_column_radiative_equilibrium_gray + # artifact_paths: "single_column_radiative_equilibrium_gray/output_active/*" + + # - label: ":computer: single column radiative equilibrium clearsky" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky.yml + # --job_id single_column_radiative_equilibrium_clearsky + # artifact_paths: "single_column_radiative_equilibrium_clearsky/output_active/*" + + # - label: ":computer: single column radiative equilibrium clearsky prognostic surface temperature" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky_prognostic_surface_temp.yml + # --job_id single_column_radiative_equilibrium_clearsky_prognostic_surface_temp + # artifact_paths: "single_column_radiative_equilibrium_clearsky_prognostic_surface_temp/output_active/*" + + # - label: ":computer: single column radiative equilibrium allsky idealized clouds varying insolation" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_allsky_idealized_clouds.yml + # --job_id single_column_radiative_equilibrium_allsky_idealized_clouds + # artifact_paths: "single_column_radiative_equilibrium_allsky_idealized_clouds/output_active/*" + + # - group: "Precipitation" + # steps: + # - label: ":umbrella: 1-moment precipitation sanity test single column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_precipitation_test.yml + # --job_id single_column_precipitation_test + # artifact_paths: "single_column_precipitation_test/output_active/*" + + # - group: "Gravity wave" + # steps: + + # - label: ":computer: non-orographic gravity wave parameterization unit test 3d" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_3d.jl" + # artifact_paths: "nonorographic_gravity_wave_test_3d/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: non-orographic gravity wave parameterization test with MiMA output" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_mima.jl" + # artifact_paths: "nonorographic_gravity_wave_test_mima/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: non-orographic gravity wave parameterization unit test single column" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_single_column.jl" + # artifact_paths: "nonorographic_gravity_wave_test_single_column/output_active/*" + + # - label: ":computer: orographic gravity wave parameterization unit test for base flux calculation" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_baseflux.jl" + # artifact_paths: "orographic_gravity_wave_test_baseflux/output_active/*" + + # - label: ":computer: orographic gravity wave parameterization unit test for 3d calculation" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_3d.jl" + # artifact_paths: "orographic_gravity_wave_test_3d/output_active/*" + + # - label: ":computer: single column non-orographic gravity wave parameterization" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_nonorographic_gravity_wave.yml + # --job_id single_column_nonorographic_gravity_wave + # artifact_paths: "single_column_nonorographic_gravity_wave/output_active/*" + + # - group: "Column Examples" + # steps: + + # - label: ":computer: single column hydrostatic balance float64" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_hydrostatic_balance_ft64.yml + # --job_id single_column_hydrostatic_balance_ft64 + # artifact_paths: "single_column_hydrostatic_balance_ft64/output_active/*" + + # - group: "Box Examples" + # steps: + + # - label: ":computer: Box hydrostatic balance (ρe_tot)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/box_hydrostatic_balance_rhoe.yml + # --job_id box_hydrostatic_balance_rhoe + # artifact_paths: "box_hydrostatic_balance_rhoe/output_active/*" + + # - label: ":computer: 3D density current" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/box_density_current_test.yml + # --job_id box_density_current_test + # artifact_paths: "box_density_current_test/output_active/*" + + # - group: "Plane Examples" + # steps: + # - label: ":computer: Agnesi linear hydrostatic mountain experiment (uniform)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_agnesi_mountain_test_uniform.yml + # --job_id plane_agnesi_mountain_test_uniform + # artifact_paths: "plane_agnesi_mountain_test_uniform/output_active/*" + + # - label: ":computer: Agnesi linear hydrostatic mountain experiment (stretched)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_agnesi_mountain_test_stretched.yml + # --job_id plane_agnesi_mountain_test_stretched + # artifact_paths: "plane_agnesi_mountain_test_stretched/output_active/*" + + # - label: ":computer: Schar mountain experiment (uniform grid)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_schar_mountain_test_uniform.yml + # --job_id plane_schar_mountain_test_uniform + # artifact_paths: "plane_schar_mountain_test_uniform/output_active/*" + + # - label: ":computer: Schar mountain experiment (stretched grid)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_schar_mountain_test_stretched.yml + # --job_id plane_schar_mountain_test_stretched + # artifact_paths: "plane_schar_mountain_test_stretched/output_active/*" + + # - label: ":computer: Density current experiment" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_density_current_test.yml + # --job_id plane_density_current_test + # artifact_paths: "plane_density_current_test/output_active/*" + + + # - group: "Sphere Examples (Dycore)" + # steps: + + # - label: ":computer: hydrostatic balance (ρe) float64" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_hydrostatic_balance_rhoe_ft64.yml + # --job_id sphere_hydrostatic_balance_rhoe_ft64 + # artifact_paths: "sphere_hydrostatic_balance_rhoe_ft64/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: baroclinic wave (ρe)" + # key: sphere_baroclinic_wave_rhoe + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe.yml + # --job_id sphere_baroclinic_wave_rhoe + # artifact_paths: "sphere_baroclinic_wave_rhoe/output_active/*" + + # - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist.yml + # --job_id sphere_baroclinic_wave_rhoe_equilmoist + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id sphere_baroclinic_wave_rhoe_equilmoist + # --out_dir sphere_baroclinic_wave_rhoe_equilmoist/output_active + # artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" + # agents: + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist (deep sphere)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml + # --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist + # --out_dir deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active + # artifact_paths: "deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" + # agents: + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist with cloud diagnostics" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist_cldiag.yml + # --job_id sphere_baroclinic_wave_rhoe_equilmoist_cldiag + # artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist_cldiag/output_active/*" + + # - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist explicit vertdiff" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist_expvdiff.yml + # --job_id sphere_baroclinic_wave_rhoe_equilmoist_expvdiff + # artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist_expvdiff/output_active/*" + + # - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist implicit vertdiff" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist_impvdiff.yml + # --job_id sphere_baroclinic_wave_rhoe_equilmoist_impvdiff + # artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist_impvdiff/output_active/*" + + # # Add this back when we figure out what to do with SSP and zalesak + # # - label: ":computer: SSP zalesak tracer & energy upwind baroclinic wave (ρe_tot) equilmoist" + # # command: > + # # julia --color=yes --project=examples examples/hybrid/driver.jl + # # --config_file $CONFIG_PATH/$$JOB_NAME.yml + # # --job_id JOB_NAME + # # artifact_paths: "$$JOB_NAME/output_active/*" + # # agents: + # # slurm_mem: 64GB + # # env: + # # JOB_NAME: "sphere_zalesak_upwind_tracer_energy_ssp_baroclinic_wave_rhoe_equilmoist" + + # - label: ":computer: no lim ARS baroclinic wave (ρe) equilmoist check conservation float64" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist_conservation_ft64.yml + # --job_id sphere_baroclinic_wave_rhoe_equilmoist_conservation_ft64 + # artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist_conservation_ft64/output_active/*" + + # - label: ":computer: held suarez (ρe) hightop" + # key: sphere_held_suarez_rhoe_hightop + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_hightop.yml + # --job_id sphere_held_suarez_rhoe_hightop + # artifact_paths: "sphere_held_suarez_rhoe_hightop/output_active/*" + + # - label: ":computer: no lim ARS held suarez (ρe) equilmoist hightop sponge" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml + # --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge + # --out_dir sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active + # artifact_paths: "sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active/*" + # agents: + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - group: "Sphere Examples (Aquaplanet)" + # steps: + + # - label: ":umbrella: aquaplanet (ρe_tot) equilmoist allsky radiation monin_obukhov varying insolation gravity wave (gfdl_restart) high top with 1-moment micro" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res.yml + # --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res + # artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: aquaplanet (ρe_tot) nonequilmoist allsky radiation monin_obukhov varying insolation gravity wave (gfdl_restart) high top with 1-moment micro" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_nonequilmoist_allsky_gw_res.yml + # --job_id sphere_aquaplanet_rhoe_nonequilmoist_allsky_gw_res + # artifact_paths: "sphere_aquaplanet_rhoe_nonequilmoist_allsky_gw_res/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: aquaplanet (ρe_tot) equilmoist allsky radiation monin_obukhov varying insolation gravity wave (raw_topo) high top zonally asymmetric" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml + # --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric + # --out_dir sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active + # artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":computer: aquaplanet (ρe_tot) slabocean equilmoist allsky radiation monin_obukhov varying insolation" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean.yml + # --job_id aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean + # artifact_paths: "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean/output_active/*" + # agents: + # slurm_mem: 20GB + + # - group: "Sphere Examples (Topography)" + # steps: + + # - label: ":computer: baroclinic wave (ρe) topography (dcmip)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_topography_dcmip_rs.yml + # --job_id sphere_baroclinic_wave_rhoe_topography_dcmip_rs + # artifact_paths: "sphere_baroclinic_wave_rhoe_topography_dcmip_rs/output_active/*" + + # - label: ":computer: held suarez (ρe) topography (dcmip)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_topography_dcmip.yml + # --job_id sphere_held_suarez_rhoe_topography_dcmip + # artifact_paths: "sphere_held_suarez_rhoe_topography_dcmip/output_active/*" + + # - label: ":computer: held suarez (ρe) equilmoist topography (dcmip)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_topography_dcmip.yml + # --job_id sphere_held_suarez_rhoe_equilmoist_topography_dcmip + # artifact_paths: "sphere_held_suarez_rhoe_equilmoist_topography_dcmip/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: Diagnostic DCMIP200 surface elevation spectra" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200.yml + # --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200 + # artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200/output_active/*" + + # - label: ":computer: Diagnostic Earth surface elevation spectra" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth.yml + # --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth + # artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth/output_active/*" + + # - group: "MPI Examples" + # steps: + + # - label: ":computer: Prep restart for MPI" + # key: "mpi_baro_wave_make_restart" + # command: > + # srun julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $MPI_CONFIG_PATH/mpi_make_restart.yml + # --job_id mpi_make_restart + # artifact_paths: "mpi_make_restart/output_active/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + + # - label: ":computer: Test restart for MPI baroclinic wave" + # key: "restart_mpi_baro_wave" + # depends_on: "mpi_baro_wave_make_restart" + # command: > + # tar xvf mpi_make_restart/output_active/hdf5_files.tar -C mpi_make_restart/output_active + + # srun julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $MPI_CONFIG_PATH/restart_mpi_baroclinic_wave_rhoe.yml + # --job_id restart_mpi_baroclinic_wave_rhoe + # artifact_paths: "restart_mpi_baroclinic_wave_rhoe/output_active/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + # timeout_in_minutes: 20 + # soft_fail: true + # #retry: + # # automatic: true + + # - label: ":computer: MPI no lim ARS aquaplanet (ρe) equilmoist clearsky radiation" + # command: > + # srun julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $MPI_CONFIG_PATH/mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky.yml + # --job_id mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky + # artifact_paths: "mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky/output_active/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: ":computer: Prep for calling remap pipeline" + # key: "prep_remap" + # command: > + # srun julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $MPI_CONFIG_PATH/prep_remap.yml + # --job_id prep_remap + # artifact_paths: "prep_remap/output_active/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + + # - label: ":computer: Exercise remap pipeline" + # key: "remap_pipeline" + # depends_on: "prep_remap" + # command: > + # tar xvf prep_remap/output_active/hdf5_files.tar -C prep_remap/output_active + + # julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl + # --data_dir prep_remap/output_active --out_dir remap_pipeline_output + # artifact_paths: "remap_pipeline_output/*" + + # - group: "Configs" + # steps: + + # - label: ":computer: Test IO" + # command: > + # julia --color=yes --project=examples --threads=8 examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/test_io.yml + # --job_id test_io + # artifact_paths: "test_io/output_active/*" + # agents: + # slurm_cpus_per_task: 8 + + # - label: ":computer: MPI io test" + # command: > + # srun julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/test_mpi_io.yml + # --job_id test_mpi_io + # artifact_paths: "test_mpi_io/output_active/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + + # # TODO: we should somehow decouple this unit test from the perf env / scripts + # - label: ":computer: checkbounds" + # command: > + # julia --color=yes --check-bounds=yes --project=perf perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/checkbounds.yml + # --job_id checkbounds + # artifact_paths: "checkbounds/output_active/*" + # agents: + # slurm_mem: 20GB + + # - group: "Diagnostic EDMFX" + # steps: + + # - label: ":genie: Diagnostic EDMFX test in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_test_box.yml + # --job_id diagnostic_edmfx_test_box + # artifact_paths: "diagnostic_edmfx_test_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX GABLS in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_gabls_box.yml + # --job_id diagnostic_edmfx_gabls_box + # artifact_paths: "diagnostic_edmfx_gabls_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX Bomex in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_box.yml + # --job_id diagnostic_edmfx_bomex_box + # artifact_paths: "diagnostic_edmfx_bomex_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX Bomex stretched grid in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_stretched_box.yml + # --job_id diagnostic_edmfx_bomex_stretched_box + # artifact_paths: "diagnostic_edmfx_bomex_stretched_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box (explicit)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_explicit_box.yml + # --job_id diagnostic_edmfx_dycoms_rf01_explicit_box + # artifact_paths: "diagnostic_edmfx_dycoms_rf01_explicit_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_box.yml + # --job_id diagnostic_edmfx_dycoms_rf01_box + # artifact_paths: "diagnostic_edmfx_dycoms_rf01_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX Rico in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_rico_box.yml + # --job_id diagnostic_edmfx_rico_box + # artifact_paths: "diagnostic_edmfx_rico_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX TRMM in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box.yml + # --job_id diagnostic_edmfx_trmm_box + # artifact_paths: "diagnostic_edmfx_trmm_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX TRMM stretched grid in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_stretched_box.yml + # --job_id diagnostic_edmfx_trmm_stretched_box + # artifact_paths: "diagnostic_edmfx_trmm_stretched_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX TRMM 0M in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box_0M.yml + # --job_id diagnostic_edmfx_trmm_box_0M + # artifact_paths: "diagnostic_edmfx_trmm_box_0M/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX aquaplanet with TKE" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml + # --job_id diagnostic_edmfx_aquaplanet + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id diagnostic_edmfx_aquaplanet + # --out_dir diagnostic_edmfx_aquaplanet/output_active + # artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc - group: "Prognostic EDMFX" steps: @@ -728,221 +728,221 @@ steps: agents: slurm_mem: 20GB - - group: "GPU" - steps: - - - label: "GPU: baroclinic wave" - key: "gpu_baroclinic_wave_rhoe" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/gpu_baroclinic_wave_rhoe.yml - --job_id gpu_baroclinic_wave_rhoe - artifact_paths: "gpu_implicit_barowave_ref/output_active/*" - agents: - slurm_mem: 16G - slurm_gpus: 1 - - - label: "GPU: compare BW with CPU" - command: > - tar xvf sphere_baroclinic_wave_rhoe/output_active/hdf5_files.tar -C sphere_baroclinic_wave_rhoe - - tar xvf gpu_baroclinic_wave_rhoe/output_active/hdf5_files.tar -C gpu_baroclinic_wave_rhoe - - julia --color=yes --project=examples post_processing/compare_outputs.jl - --output_folder_1 sphere_baroclinic_wave_rhoe - --output_folder_2 gpu_baroclinic_wave_rhoe --t_end 10days - --compare_state false - depends_on: - - "sphere_baroclinic_wave_rhoe" - - "gpu_baroclinic_wave_rhoe" - - - label: "GPU: GPU dry baroclinic wave" - key: "target_gpu_implicit_baroclinic_wave" - command: - - mkdir -p target_gpu_implicit_baroclinic_wave - - > - nsys profile --trace=nvtx,cuda --output=target_gpu_implicit_baroclinic_wave/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave.yml - --job_id target_gpu_implicit_baroclinic_wave - artifact_paths: "target_gpu_implicit_baroclinic_wave/output_active/*" - agents: - slurm_gpus: 1 - slurm_mem: 32G - - - label: "GPU: GPU dry baroclinic wave - 4 gpus" - key: "target_gpu_implicit_baroclinic_wave_4process" - command: - # nsys profile --trace=nvtx,cuda,mpi --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK} - - mkdir -p target_gpu_implicit_baroclinic_wave_4process - - > - srun --cpu-bind=threads --cpus-per-task=4 - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl - --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave_4process.yml - --job_id target_gpu_implicit_baroclinic_wave_4process - artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/output_active/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_gpus_per_task: 1 - slurm_cpus_per_task: 4 - slurm_ntasks: 4 - slurm_mem: 32G - - - label: "GPU: GPU moist Held-Suarez" - command: - - > - nsys profile --trace=nvtx,cuda --output=central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/central_gpu_hs_rhoe_equil_55km_nz63_0M.yml - --job_id central_gpu_hs_rhoe_equil_55km_nz63_0M - artifact_paths: "central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" - agents: - slurm_gpus: 1 - slurm_mem: 16G - - - label: "GPU: GPU moist Held-Suarez cloud diagnostics per stage" - command: - - > - nsys profile --trace=nvtx,cuda --output=central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M.yml - --job_id central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M - artifact_paths: "central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" - agents: - slurm_gpus: 1 - slurm_mem: 16G - - - label: ":umbrella: GPU: gpu_aquaplanet_dyamond" - command: - - mkdir -p gpu_aquaplanet_dyamond - - > - nsys profile --trace=nvtx,cuda --output=gpu_aquaplanet_dyamond/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file ${CONFIG_PATH}/gpu_aquaplanet_dyamond.yml - --job_id gpu_aquaplanet_dyamond - artifact_paths: "gpu_aquaplanet_dyamond/output_active/*" - agents: - slurm_gpus: 1 - - - label: "GPU: Diagnostic EDMFX aquaplanet" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/gpu_diagnostic_edmfx_aquaplanet.yml - --job_id gpu_diagnostic_edmfx_aquaplanet - artifact_paths: "gpu_diagnostic_edmfx_aquaplanet/output_active/*" - agents: - slurm_gpus: 1 - slurm_mem: 20G - - - label: "GPU: Prognostic EDMFX aquaplanet" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/gpu_prognostic_edmfx_aquaplanet.yml - --job_id gpu_prognostic_edmfx_aquaplanet - artifact_paths: "gpu_prognostic_edmfx_aquaplanet/output_active/*" - agents: - slurm_gpus: 1 - slurm_mem: 20G - - - group: "GPU Performance" - steps: - - - label: "Perf: GPU implicit baro wave wrt h_elem" - key: "gpu_implicit_barowave_wrt_h_elem" - command: > - - julia --color=yes --project=examples perf/benchmark_dump.jl - --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave_wrt_h_elem.yml - --job_id gpu_implicit_barowave_wrt_h_elem - artifact_paths: "gpu_implicit_barowave_wrt_h_elem/output_active/*" - agents: - slurm_gpus: 1 - - - label: "Perf: GPU implicit baro wave" - command: > - - julia --color=yes --project=examples perf/benchmark_step.jl - --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave.yml - --job_id gpu_implicit_barowave - artifact_paths: "gpu_implicit_barowave/output_active/*" - agents: - slurm_gpus: 1 - - - label: "Perf: GPU implicit baro wave moist" - command: > - - julia --color=yes --project=examples perf/benchmark_step.jl - --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave_moist.yml - --job_id gpu_implicit_barowave_moist - artifact_paths: "gpu_implicit_barowave_moist/output_active/*" - agents: - slurm_mem: 16G - slurm_gpus: 1 - - - label: "Perf: CPU implicit baro wave" - command: > - julia --color=yes --project=examples perf/benchmark_step.jl - --config_file $PERF_CONFIG_PATH/cpu_implicit_barowave.yml - --job_id cpu_implicit_barowave - artifact_paths: "cpu_implicit_barowave/output_active/*" + # - group: "GPU" + # steps: + + # - label: "GPU: baroclinic wave" + # key: "gpu_baroclinic_wave_rhoe" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/gpu_baroclinic_wave_rhoe.yml + # --job_id gpu_baroclinic_wave_rhoe + # artifact_paths: "gpu_implicit_barowave_ref/output_active/*" + # agents: + # slurm_mem: 16G + # slurm_gpus: 1 + + # - label: "GPU: compare BW with CPU" + # command: > + # tar xvf sphere_baroclinic_wave_rhoe/output_active/hdf5_files.tar -C sphere_baroclinic_wave_rhoe + + # tar xvf gpu_baroclinic_wave_rhoe/output_active/hdf5_files.tar -C gpu_baroclinic_wave_rhoe + + # julia --color=yes --project=examples post_processing/compare_outputs.jl + # --output_folder_1 sphere_baroclinic_wave_rhoe + # --output_folder_2 gpu_baroclinic_wave_rhoe --t_end 10days + # --compare_state false + # depends_on: + # - "sphere_baroclinic_wave_rhoe" + # - "gpu_baroclinic_wave_rhoe" + + # - label: "GPU: GPU dry baroclinic wave" + # key: "target_gpu_implicit_baroclinic_wave" + # command: + # - mkdir -p target_gpu_implicit_baroclinic_wave + # - > + # nsys profile --trace=nvtx,cuda --output=target_gpu_implicit_baroclinic_wave/output_active/report + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave.yml + # --job_id target_gpu_implicit_baroclinic_wave + # artifact_paths: "target_gpu_implicit_baroclinic_wave/output_active/*" + # agents: + # slurm_gpus: 1 + # slurm_mem: 32G + + # - label: "GPU: GPU dry baroclinic wave - 4 gpus" + # key: "target_gpu_implicit_baroclinic_wave_4process" + # command: + # # nsys profile --trace=nvtx,cuda,mpi --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK} + # - mkdir -p target_gpu_implicit_baroclinic_wave_4process + # - > + # srun --cpu-bind=threads --cpus-per-task=4 + # julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + # --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave_4process.yml + # --job_id target_gpu_implicit_baroclinic_wave_4process + # artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/output_active/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_gpus_per_task: 1 + # slurm_cpus_per_task: 4 + # slurm_ntasks: 4 + # slurm_mem: 32G + + # - label: "GPU: GPU moist Held-Suarez" + # command: + # - > + # nsys profile --trace=nvtx,cuda --output=central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/central_gpu_hs_rhoe_equil_55km_nz63_0M.yml + # --job_id central_gpu_hs_rhoe_equil_55km_nz63_0M + # artifact_paths: "central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16G + + # - label: "GPU: GPU moist Held-Suarez cloud diagnostics per stage" + # command: + # - > + # nsys profile --trace=nvtx,cuda --output=central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M.yml + # --job_id central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M + # artifact_paths: "central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16G + + # - label: ":umbrella: GPU: gpu_aquaplanet_dyamond" + # command: + # - mkdir -p gpu_aquaplanet_dyamond + # - > + # nsys profile --trace=nvtx,cuda --output=gpu_aquaplanet_dyamond/output_active/report + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file ${CONFIG_PATH}/gpu_aquaplanet_dyamond.yml + # --job_id gpu_aquaplanet_dyamond + # artifact_paths: "gpu_aquaplanet_dyamond/output_active/*" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Diagnostic EDMFX aquaplanet" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/gpu_diagnostic_edmfx_aquaplanet.yml + # --job_id gpu_diagnostic_edmfx_aquaplanet + # artifact_paths: "gpu_diagnostic_edmfx_aquaplanet/output_active/*" + # agents: + # slurm_gpus: 1 + # slurm_mem: 20G + + # - label: "GPU: Prognostic EDMFX aquaplanet" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/gpu_prognostic_edmfx_aquaplanet.yml + # --job_id gpu_prognostic_edmfx_aquaplanet + # artifact_paths: "gpu_prognostic_edmfx_aquaplanet/output_active/*" + # agents: + # slurm_gpus: 1 + # slurm_mem: 20G + + # - group: "GPU Performance" + # steps: + + # - label: "Perf: GPU implicit baro wave wrt h_elem" + # key: "gpu_implicit_barowave_wrt_h_elem" + # command: > + + # julia --color=yes --project=examples perf/benchmark_dump.jl + # --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave_wrt_h_elem.yml + # --job_id gpu_implicit_barowave_wrt_h_elem + # artifact_paths: "gpu_implicit_barowave_wrt_h_elem/output_active/*" + # agents: + # slurm_gpus: 1 + + # - label: "Perf: GPU implicit baro wave" + # command: > + + # julia --color=yes --project=examples perf/benchmark_step.jl + # --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave.yml + # --job_id gpu_implicit_barowave + # artifact_paths: "gpu_implicit_barowave/output_active/*" + # agents: + # slurm_gpus: 1 + + # - label: "Perf: GPU implicit baro wave moist" + # command: > + + # julia --color=yes --project=examples perf/benchmark_step.jl + # --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave_moist.yml + # --job_id gpu_implicit_barowave_moist + # artifact_paths: "gpu_implicit_barowave_moist/output_active/*" + # agents: + # slurm_mem: 16G + # slurm_gpus: 1 + + # - label: "Perf: CPU implicit baro wave" + # command: > + # julia --color=yes --project=examples perf/benchmark_step.jl + # --config_file $PERF_CONFIG_PATH/cpu_implicit_barowave.yml + # --job_id cpu_implicit_barowave + # artifact_paths: "cpu_implicit_barowave/output_active/*" - group: "Performance" steps: - # Benchmarks - - label: ":computer: Benchmark: perf target (default)" - command: > - julia --color=yes --project=perf perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_perf_target.yml - --job_id bm_perf_target - agents: - slurm_mem: 24GB + # # Benchmarks + # - label: ":computer: Benchmark: perf target (default)" + # command: > + # julia --color=yes --project=perf perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_perf_target.yml + # --job_id bm_perf_target + # agents: + # slurm_mem: 24GB - - label: ":computer: Benchmark: perf target (gpu)" - command: > - julia --color=yes --project=perf perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_perf_target.yml - --job_id bm_perf_target_gpu - agents: - slurm_mem: 24GB - slurm_gpus: 1 + # - label: ":computer: Benchmark: perf target (gpu)" + # command: > + # julia --color=yes --project=perf perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_perf_target.yml + # --job_id bm_perf_target_gpu + # agents: + # slurm_mem: 24GB + # slurm_gpus: 1 - - label: ":computer: Benchmark: perf target (Threaded)" - command: > - julia --color=yes --threads 8 --project=perf perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_perf_target_threaded.yml - --job_id bm_perf_target_threaded - agents: - slurm_mem: 24GB - slurm_cpus_per_task: 8 + # - label: ":computer: Benchmark: perf target (Threaded)" + # command: > + # julia --color=yes --threads 8 --project=perf perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_perf_target_threaded.yml + # --job_id bm_perf_target_threaded + # agents: + # slurm_mem: 24GB + # slurm_cpus_per_task: 8 - - label: ":fire: Flame graph: perf target (default)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target.yml - --job_id flame_perf_target - artifact_paths: "flame_perf_target/*" - agents: - slurm_mem: 24GB + # - label: ":fire: Flame graph: perf target (default)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target.yml + # --job_id flame_perf_target + # artifact_paths: "flame_perf_target/*" + # agents: + # slurm_mem: 24GB - - label: ":fire: Flame graph: perf target (with tracers)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_tracers.yml - --job_id flame_perf_target_tracers - artifact_paths: "flame_perf_target_tracers/*" - agents: - slurm_mem: 24GB + # - label: ":fire: Flame graph: perf target (with tracers)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_tracers.yml + # --job_id flame_perf_target_tracers + # artifact_paths: "flame_perf_target_tracers/*" + # agents: + # slurm_mem: 24GB - - label: ":fire: Flame graph: perf target (diagnostic edmfx)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_diagnostic_edmfx.yml - --job_id flame_perf_target_diagnostic_edmfx - artifact_paths: "flame_perf_target_diagnostic_edmfx/*" - agents: - slurm_mem: 24GB + # - label: ":fire: Flame graph: perf target (diagnostic edmfx)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_diagnostic_edmfx.yml + # --job_id flame_perf_target_diagnostic_edmfx + # artifact_paths: "flame_perf_target_diagnostic_edmfx/*" + # agents: + # slurm_mem: 24GB - label: ":fire: Flame graph: perf target (prognostic edmfx aquaplanet)" command: > @@ -953,88 +953,88 @@ steps: agents: slurm_mem: 48GB - - label: ":fire: Flame graph: perf target (barowave jfnk)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_sphere_baroclinic_wave_rhoe_equilmoist_expvdiff.yml - --job_id flame_sphere_baroclinic_wave_rhoe_equilmoist_expvdiff - artifact_paths: "flame_sphere_baroclinic_wave_rhoe_equilmoist_expvdiff/*" - agents: - slurm_mem: 40GB + # - label: ":fire: Flame graph: perf target (barowave jfnk)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_sphere_baroclinic_wave_rhoe_equilmoist_expvdiff.yml + # --job_id flame_sphere_baroclinic_wave_rhoe_equilmoist_expvdiff + # artifact_paths: "flame_sphere_baroclinic_wave_rhoe_equilmoist_expvdiff/*" + # agents: + # slurm_mem: 40GB - - label: ":fire: Flame graph: perf target (frierson diffusion)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_frierson.yml - --job_id flame_perf_target_frierson - artifact_paths: "flame_perf_target_frierson/*" - agents: - slurm_mem: 48GB + # - label: ":fire: Flame graph: perf target (frierson diffusion)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_frierson.yml + # --job_id flame_perf_target_frierson + # artifact_paths: "flame_perf_target_frierson/*" + # agents: + # slurm_mem: 48GB - - label: ":fire: Flame graph: perf target (Threaded)" - command: > - julia --threads 8 --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_threaded.yml - --job_id flame_perf_target_threaded - artifact_paths: "flame_perf_target_threaded/*" - agents: - slurm_cpus_per_task: 8 - slurm_mem: 24GB + # - label: ":fire: Flame graph: perf target (Threaded)" + # command: > + # julia --threads 8 --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_threaded.yml + # --job_id flame_perf_target_threaded + # artifact_paths: "flame_perf_target_threaded/*" + # agents: + # slurm_cpus_per_task: 8 + # slurm_mem: 24GB - - label: ":fire: Flame graph: perf target (Callbacks)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_callbacks.yml - --job_id flame_perf_target_callbacks - artifact_paths: "flame_perf_target_callbacks/*" - agents: - slurm_mem: 24GB + # - label: ":fire: Flame graph: perf target (Callbacks)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_callbacks.yml + # --job_id flame_perf_target_callbacks + # artifact_paths: "flame_perf_target_callbacks/*" + # agents: + # slurm_mem: 24GB - - label: ":fire: Flame graph: gravity wave" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_gw.yml - --job_id flame_perf_gw - artifact_paths: "flame_perf_gw/*" - agents: - slurm_mem: 24GB + # - label: ":fire: Flame graph: gravity wave" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_gw.yml + # --job_id flame_perf_gw + # artifact_paths: "flame_perf_gw/*" + # agents: + # slurm_mem: 24GB - - label: ":fire: Flame graph: perf target (diagnostics)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_diagnostics.yml - --job_id flame_perf_diagnostics - artifact_paths: "flame_perf_diagnostics/*" - agents: - slurm_mem: 24GB + # - label: ":fire: Flame graph: perf target (diagnostics)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_diagnostics.yml + # --job_id flame_perf_diagnostics + # artifact_paths: "flame_perf_diagnostics/*" + # agents: + # slurm_mem: 24GB - - label: ":fire: Flame graph: gpu job" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_gpu_implicit_barowave_moist.yml - --job_id flame_gpu_implicit_barowave_moist - artifact_paths: "flame_gpu_implicit_barowave_moist/*" - agents: - slurm_mem: 48GB - slurm_gpus: 1 - gres: "gpu:p100:1" + # - label: ":fire: Flame graph: gpu job" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_gpu_implicit_barowave_moist.yml + # --job_id flame_gpu_implicit_barowave_moist + # artifact_paths: "flame_gpu_implicit_barowave_moist/*" + # agents: + # slurm_mem: 48GB + # slurm_gpus: 1 + # gres: "gpu:p100:1" - # Inference - - label: ":rocket: JET n-failures (inference)" - command: > - julia --color=yes --project=perf perf/jet_test_nfailures.jl - --config_file $PERF_CONFIG_PATH/jet_n_failures.yml - --job_id jet_n_failures - agents: - slurm_mem: 24GB + # # Inference + # - label: ":rocket: JET n-failures (inference)" + # command: > + # julia --color=yes --project=perf perf/jet_test_nfailures.jl + # --config_file $PERF_CONFIG_PATH/jet_n_failures.yml + # --job_id jet_n_failures + # agents: + # slurm_mem: 24GB - # Latency - - label: ":mag::rocket: Invalidations" - command: > - julia --color=yes --project=perf perf/invalidations.jl - artifact_paths: "invalidations/*" - agents: - slurm_mem: 24GB + # # Latency + # - label: ":mag::rocket: Invalidations" + # command: > + # julia --color=yes --project=perf perf/invalidations.jl + # artifact_paths: "invalidations/*" + # agents: + # slurm_mem: 24GB - wait: ~ continue_on_failure: true diff --git a/config/longrun_configs/longrun_aquaplanet_equil_helem12_allsky_progedmf_0M.yml b/config/longrun_configs/longrun_aquaplanet_equil_helem12_allsky_progedmf_0M.yml new file mode 100644 index 00000000000..bb89233dc76 --- /dev/null +++ b/config/longrun_configs/longrun_aquaplanet_equil_helem12_allsky_progedmf_0M.yml @@ -0,0 +1,38 @@ +h_elem: 12 +z_max: 55000.0 +z_elem: 30 +dz_bottom: 100.0 +dz_top: 3000.0 +moist: "equil" +precip_model: "0M" +override_τ_precip: false +rad: "allskywithclear" +dt_rad: "6hours" +surface_setup: "DefaultMoninObukhov" +turbconv: "prognostic_edmfx" +implicit_sgs_advection: true +implicit_diffusion: true +approximate_linear_solve_iters: 2 +max_newton_iters_ode: 3 +prognostic_tke: true +edmfx_upwinding: "first_order" +edmfx_entr_model: "Generalized" +edmfx_detr_model: "Generalized" +edmfx_nh_pressure: true +edmfx_filter: true +edmfx_sgs_mass_flux: true +edmfx_sgs_diffusive_flux: true +rayleigh_sponge: true +restart_file: "/central/groups/esm/zhaoyi/climaatmos/longrun_1560/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M/output_active/day30.0.hdf5" +dt_save_state_to_disk: "10days" +dt: "20secs" +t_end: "60days" +toml: [toml/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.toml] +netcdf_output_at_levels: true +diagnostics: + - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl] + period: 1days + - short_name: [arup, waup, rhoaup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke] + period: 1days + - short_name: [entr, detr, lmix, bgrad, strain, edt, evu] + period: 1days diff --git a/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M.yml b/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M.yml new file mode 100644 index 00000000000..ad6b9145e25 --- /dev/null +++ b/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M.yml @@ -0,0 +1,23 @@ +h_elem: 12 +z_max: 55000.0 +z_elem: 30 +dz_bottom: 100.0 +dz_top: 3000.0 +moist: "equil" +precip_model: "0M" +override_τ_precip: false +rad: "clearsky" +dt_rad: "6hours" +surface_setup: "DefaultMoninObukhov" +vert_diff: "FriersonDiffusion" +rayleigh_sponge: true +dt_save_state_to_disk: "10days" +dt: "200secs" +t_end: "100days" +toml: [toml/longrun_aquaplanet_equil_helem12_clearsky_0M.toml] +netcdf_output_at_levels: true +diagnostics: + - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl] + period: 10days + - short_name: [lmix, bgrad, strain, edt, evu] + period: 10days diff --git a/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.yml b/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.yml new file mode 100644 index 00000000000..c63e5a810a3 --- /dev/null +++ b/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.yml @@ -0,0 +1,24 @@ +h_elem: 12 +z_max: 55000.0 +z_elem: 30 +dz_bottom: 100.0 +dz_top: 3000.0 +moist: "equil" +precip_model: "0M" +override_τ_precip: false +rad: "clearsky" +dt_rad: "6hours" +surface_setup: "DefaultMoninObukhov" +vert_diff: "FriersonDiffusion" +rayleigh_sponge: true +viscous_sponge: true +dt_save_state_to_disk: "10days" +dt: "200secs" +t_end: "100days" +toml: [toml/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.toml] +netcdf_output_at_levels: true +diagnostics: + - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl] + period: 10days + - short_name: [lmix, bgrad, strain, edt, evu] + period: 10days diff --git a/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.yml b/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.yml new file mode 100644 index 00000000000..19300be151f --- /dev/null +++ b/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.yml @@ -0,0 +1,39 @@ +h_elem: 12 +z_max: 55000.0 +z_elem: 30 +dz_bottom: 100.0 +dz_top: 3000.0 +moist: "equil" +precip_model: "0M" +override_τ_precip: false +rad: "clearsky" +dt_rad: "6hours" +surface_setup: "DefaultMoninObukhov" +turbconv: "prognostic_edmfx" +implicit_sgs_advection: true +implicit_diffusion: true +approximate_linear_solve_iters: 2 +max_newton_iters_ode: 3 +prognostic_tke: true +edmfx_upwinding: "first_order" +edmfx_entr_model: "Generalized" +edmfx_detr_model: "Generalized" +edmfx_nh_pressure: true +edmfx_filter: true +edmfx_sgs_mass_flux: true +edmfx_sgs_diffusive_flux: true +rayleigh_sponge: true +viscous_sponge: true +restart_file: "/central/groups/esm/zhaoyi/climaatmos/longrun_1560/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M/output_active/day30.0.hdf5" +dt_save_state_to_disk: "10days" +dt: "20secs" +t_end: "32days" +toml: [toml/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.toml] +netcdf_output_at_levels: true +diagnostics: + - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl] + period: 1days + - short_name: [arup, waup, rhoaup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke] + period: 1days + - short_name: [entr, detr, lmix, bgrad, strain, edt, evu] + period: 1days diff --git a/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M_asym.yml b/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M_asym.yml new file mode 100644 index 00000000000..4c92e1e9609 --- /dev/null +++ b/config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M_asym.yml @@ -0,0 +1,38 @@ +h_elem: 12 +z_max: 55000.0 +z_elem: 30 +dz_bottom: 100.0 +dz_top: 3000.0 +moist: "equil" +precip_model: "0M" +override_τ_precip: false +surface_temperature: "ZonallyAsymmetric" +rad: "clearsky" +dt_rad: "6hours" +surface_setup: "DefaultMoninObukhov" +turbconv: "prognostic_edmfx" +implicit_sgs_advection: true +implicit_diffusion: true +approximate_linear_solve_iters: 2 +max_newton_iters_ode: 3 +prognostic_tke: true +edmfx_upwinding: "first_order" +edmfx_entr_model: "Generalized" +edmfx_detr_model: "Generalized" +edmfx_nh_pressure: true +edmf_filter: true +edmfx_sgs_mass_flux: true +edmfx_sgs_diffusive_flux: true +rayleigh_sponge: true +dt_save_state_to_disk: "10days" +dt: "40secs" +t_end: "100days" +toml: [toml/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.toml] +netcdf_output_at_levels: true +diagnostics: + - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl] + period: 10days + - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke] + period: 10days + - short_name: [entr, detr, lmix, bgrad, strain, edt, evu] + period: 10days diff --git a/config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.yml b/config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.yml new file mode 100644 index 00000000000..e9670daf507 --- /dev/null +++ b/config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.yml @@ -0,0 +1,37 @@ +h_elem: 16 +z_max: 55000.0 +z_elem: 63 +dz_bottom: 30.0 +dz_top: 3000.0 +moist: "equil" +precip_model: "0M" +override_τ_precip: false +rad: "clearsky" +dt_rad: "6hours" +surface_setup: "DefaultMoninObukhov" +turbconv: "prognostic_edmfx" +implicit_sgs_advection: true +implicit_diffusion: true +approximate_linear_solve_iters: 2 +max_newton_iters_ode: 3 +prognostic_tke: true +edmfx_upwinding: "first_order" +edmfx_entr_model: "Generalized" +edmfx_detr_model: "Generalized" +edmfx_nh_pressure: true +edmf_filter: true +edmfx_sgs_mass_flux: true +edmfx_sgs_diffusive_flux: true +rayleigh_sponge: true +dt_save_state_to_disk: "3hours" +dt: "40secs" +t_end: "10days" +toml: [toml/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.toml] +netcdf_output_at_levels: true +diagnostics: + - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl] + period: 1days + - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke] + period: 1days + - short_name: [entr, detr, lmix, bgrad, strain, edt, evu] + period: 1days diff --git a/config/model_configs/prognostic_edmfx_adv_test_column.yml b/config/model_configs/prognostic_edmfx_adv_test_column.yml index a53d91d1a50..c4856c8277c 100644 --- a/config/model_configs/prognostic_edmfx_adv_test_column.yml +++ b/config/model_configs/prognostic_edmfx_adv_test_column.yml @@ -20,5 +20,7 @@ netcdf_interpolation_num_points: [2, 2, 63] diagnostics: - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl] period: 10mins - - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke, lmix] + - short_name: [arup, waup, taup, thetaaup, haup, rhoaup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke, lmix] + period: 10mins + - short_name: [entr, detr, lmix, bgrad, strain, edt, evu] period: 10mins diff --git a/config/model_configs/prognostic_edmfx_aquaplanet.yml b/config/model_configs/prognostic_edmfx_aquaplanet.yml index f9895b9a0ae..f25d36bfdd6 100644 --- a/config/model_configs/prognostic_edmfx_aquaplanet.yml +++ b/config/model_configs/prognostic_edmfx_aquaplanet.yml @@ -17,9 +17,12 @@ dt_save_state_to_disk: 600secs toml: [toml/prognostic_edmfx.toml] output_default_diagnostics: false diagnostics: - - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hfes, hur, hus, cl, clw, cli, evspsbl, rsd, rsu, rld, rlu] + - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl] reduction_time: average period: 1hours - - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, tke, lmix] + - short_name: [arup, waup, taup, thetaaup, haup, rhoaup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke, lmix] + reduction_time: average + period: 1hours + - short_name: [entr, detr, lmix, bgrad, strain, edt, evu] reduction_time: average period: 1hours diff --git a/config/model_configs/prognostic_edmfx_trmm_column_0M.yml b/config/model_configs/prognostic_edmfx_trmm_column_0M.yml index 267a6d912ec..868d0d8a07d 100644 --- a/config/model_configs/prognostic_edmfx_trmm_column_0M.yml +++ b/config/model_configs/prognostic_edmfx_trmm_column_0M.yml @@ -32,5 +32,7 @@ netcdf_interpolation_num_points: [8, 8, 82] diagnostics: - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl] period: 10mins - - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke, lmix] + - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke] + period: 10mins + - short_name: [entr, detr, lmix, bgrad, strain, edt, evu] period: 10mins diff --git a/post_processing/ci_plots.jl b/post_processing/ci_plots.jl index afc3ce6f10b..9f7ffe14f76 100644 --- a/post_processing/ci_plots.jl +++ b/post_processing/ci_plots.jl @@ -1123,26 +1123,26 @@ function make_plots( short_names = [ "ua", - "wa", + "va", + "tke", + "lmix", + "bgrad", + "strain", "thetaa", "thetaaup", "ta", "taup", "ha", "haup", + "wa", "waup", - "tke", - "lmix", "arup", "hus", "husup", "hur", "hurup", - "cl", "clw", "clwup", - "cli", - "cliup", precip_names..., ] reduction = "inst" @@ -1174,6 +1174,13 @@ function make_plots( ([slice(v, time = LAST_SNAP) for v in group]...,) for group in var_groups_zt ] + # var_groups_z = [ + # ( + # [ + # ClimaAnalysis.window(v, "time", left = 14400, right = 21600) |> ClimaAnalysis.average_time for v in group + # ]..., + # ) for group in var_groups_zt + # ] tmp_file = make_plots_generic( output_paths, diff --git a/src/initial_conditions/initial_conditions.jl b/src/initial_conditions/initial_conditions.jl index 87dced494bc..94e30b861bc 100644 --- a/src/initial_conditions/initial_conditions.jl +++ b/src/initial_conditions/initial_conditions.jl @@ -881,6 +881,13 @@ end The `InitialCondition` described in [Brown2002](@cite), but with a hydrostatically balanced pressure profile. """ + +bomex_tke(::Type{FT}) where {FT} = z -> if (z <= 2500.0) + FT(1) - z / 3000 +else + FT(0) +end + Base.@kwdef struct ARM_SGP <: InitialCondition prognostic_tke::Bool = false end @@ -918,7 +925,7 @@ for IC in (:Soares, :Bomex, :LifeCycleTan2018, :ARM_SGP) ), velocity = Geometry.UVector(u(z)), turbconv_state = EDMFState(; - tke = prognostic_tke ? FT(0) : tke(z), + tke = prognostic_tke ? bomex_tke(FT)(z) : tke(z), ), ) end diff --git a/src/prognostic_equations/edmfx_closures.jl b/src/prognostic_equations/edmfx_closures.jl index 19d34f1b634..06ac5b459c1 100644 --- a/src/prognostic_equations/edmfx_closures.jl +++ b/src/prognostic_equations/edmfx_closures.jl @@ -110,8 +110,37 @@ function edmfx_nh_pressure_tendency!( ᶠlg = Fields.local_geometry_field(Y.f) scale_height = CAP.R_d(params) * CAP.T_surf_ref(params) / CAP.grav(params) + FT = eltype(Y) + ᶜz = Fields.coordinate_field(Y.c).z + z_sfc = Fields.level(Fields.coordinate_field(Y.f).z, Fields.half) + z_sfc_data = Fields.field_values(z_sfc[colidx]) + turbconv_params = CAP.turbconv_params(params) + a_min = CAP.min_area(turbconv_params) + ᶜupdraft_top = p.scratch.ᶜtemp_scalar for j in 1:n + # # look for updraft top + # ᶜupdraft_top_data = Fields.field_values(ᶜupdraft_top[colidx]) + # @. ᶜupdraft_top_data = FT(0) + # for level in 1:Spaces.nlevels(axes(ᶜz)) + # ρaʲ_lev = Fields.field_values( + # Spaces.level(Y.c.sgsʲs.:($j).ρa[colidx], level), + # ) + # ρʲ_lev = + # Fields.field_values(Spaces.level(ᶜρʲs.:($j)[colidx], level)) + # ᶜz_lev = Fields.field_values(Spaces.level(ᶜz[colidx], level)) + # @. ᶜupdraft_top_data = ifelse( + # draft_area(ρaʲ_lev, ρʲ_lev) > a_min, + # ᶜz_lev, + # ᶜupdraft_top_data, + # ) + # end + # @. ᶜupdraft_top_data = ᶜupdraft_top_data - z_sfc_data + + # # There's only one updraft_top per column, so it's + # # safe to use at cell centers and cell faces, correct?: + # ᶠupdraft_top = Fields.Field(ᶜupdraft_top_data, axes(ᶠu₃⁰[colidx])) + @. ᶠnh_pressure₃ʲs.:($$j)[colidx] = ᶠupdraft_nh_pressure( params, p.atmos.edmfx_nh_pressure, @@ -247,9 +276,15 @@ function mixing_length( l_TKE > l_z ? l_z : l_TKE, l_W > l_z ? l_z : l_W, ) + # l = SA.SVector( + # (l_N < eps(FT) || l_N > l_z) ? l_z : l_N, + # (l_TKE < eps(FT) || l_TKE > l_z) ? l_z : l_TKE, + # (l_W < eps(FT) || l_W > l_z) ? l_z : l_W, + # ) # get soft minimum l_smin = lamb_smooth_minimum(l, smin_ub, smin_rm) l_limited = max(l_smag, min(l_smin, l_z)) + #l_limited = min(200, l_z) return MixingLength{FT}(l_limited, l_W, l_TKE, l_N) end @@ -310,6 +345,8 @@ function edmfx_filter_tendency!( n = n_mass_flux_subdomains(turbconv_model) (; dt) = p + (; ᶜK, ᶜh_tot, ᶜspecific, ᶜρa⁰, ᶜtke⁰) = p.precomputed + ᶜu₃ʲ = p.scratch.ᶜtemp_C3 if p.atmos.edmfx_filter for j in 1:n @@ -317,6 +354,17 @@ function edmfx_filter_tendency!( C3(min(Y.f.sgsʲs.:($$j).u₃[colidx].components.data.:1, 0)) / dt @. Yₜ.c.sgsʲs.:($$j).ρa[colidx] -= min(Y.c.sgsʲs.:($$j).ρa[colidx], 0) / dt + # @. ᶜu₃ʲ[colidx] = ᶜinterp(Y.f.sgsʲs.:($$j).u₃[colidx]) + # @. Yₜ.c.sgsʲs.:($$j).mse[colidx] -= ifelse( + # ᶜu₃ʲ[colidx].components.data.:1 < 0, + # (Y.c.sgsʲs.:($$j).mse[colidx] - (ᶜh_tot[colidx] - ᶜK[colidx])) / dt, + # 0, + # ) + # @. Yₜ.c.sgsʲs.:($$j).q_tot[colidx] -= ifelse( + # ᶜu₃ʲ[colidx].components.data.:1 < 0, + # (Y.c.sgsʲs.:($$j).q_tot[colidx] - ᶜspecific.q_tot[colidx]) / dt, + # 0, + # ) end end end diff --git a/src/prognostic_equations/edmfx_sgs_flux.jl b/src/prognostic_equations/edmfx_sgs_flux.jl index 10dba82f650..9453d9a5755 100644 --- a/src/prognostic_equations/edmfx_sgs_flux.jl +++ b/src/prognostic_equations/edmfx_sgs_flux.jl @@ -14,17 +14,22 @@ function edmfx_sgs_mass_flux_tendency!( ) n = n_mass_flux_subdomains(turbconv_model) + turbconv_params = CAP.turbconv_params(p.params) + a_max = CAP.max_area(turbconv_params) (; edmfx_sgsflux_upwinding) = p.atmos.numerics (; ᶠu³, ᶜh_tot, ᶜspecific) = p.precomputed (; ᶠu³ʲs, ᶜKʲs, ᶜρʲs) = p.precomputed (; ᶜρa⁰, ᶜρ⁰, ᶠu³⁰, ᶜK⁰, ᶜmse⁰, ᶜq_tot⁰) = p.precomputed (; dt) = p ᶜJ = Fields.local_geometry_field(Y.c).J + FT = eltype(Y) + ᶜlg = Fields.local_geometry_field(Y.c) if p.atmos.edmfx_sgs_mass_flux # energy ᶠu³_diff_colidx = p.scratch.ᶠtemp_CT3[colidx] ᶜa_scalar_colidx = p.scratch.ᶜtemp_scalar[colidx] + ᶜwdata_diff_colidx = p.scratch.ᶜtemp_scalar_2[colidx] for j in 1:n @. ᶠu³_diff_colidx = ᶠu³ʲs.:($$j)[colidx] - ᶠu³[colidx] @. ᶜa_scalar_colidx = @@ -32,6 +37,22 @@ function edmfx_sgs_mass_flux_tendency!( Y.c.sgsʲs.:($$j).mse[colidx] + ᶜKʲs.:($$j)[colidx] - ᶜh_tot[colidx] ) * draft_area(Y.c.sgsʲs.:($$j).ρa[colidx], ᶜρʲs.:($$j)[colidx]) + @. ᶜwdata_diff_colidx = + get_physical_w(ᶜinterp(ᶠu³_diff_colidx), ᶜlg[colidx]) + # @. ᶜa_scalar_colidx = + # ( + # Y.c.sgsʲs.:($$j).mse[colidx] + ᶜKʲs.:($$j)[colidx] - + # ᶜh_tot[colidx] + # ) * min( + # min( + # draft_area( + # Y.c.sgsʲs.:($$j).ρa[colidx], + # ᶜρʲs.:($$j)[colidx], + # ), + # a_max, + # ), + # FT(0.02) / max(ᶜwdata_diff_colidx, $eps(FT)), + # ) vertical_transport!( Yₜ.c.ρe_tot[colidx], ᶜJ[colidx], @@ -63,6 +84,20 @@ function edmfx_sgs_mass_flux_tendency!( @. ᶜa_scalar_colidx = (Y.c.sgsʲs.:($$j).q_tot[colidx] - ᶜspecific.q_tot[colidx]) * draft_area(Y.c.sgsʲs.:($$j).ρa[colidx], ᶜρʲs.:($$j)[colidx]) + @. ᶜwdata_diff_colidx = + get_physical_w(ᶜinterp(ᶠu³_diff_colidx), ᶜlg[colidx]) + # @. ᶜa_scalar_colidx = + # (Y.c.sgsʲs.:($$j).q_tot[colidx] - ᶜspecific.q_tot[colidx]) * + # min( + # min( + # draft_area( + # Y.c.sgsʲs.:($$j).ρa[colidx], + # ᶜρʲs.:($$j)[colidx], + # ), + # a_max, + # ), + # FT(0.02) / max(ᶜwdata_diff_colidx, $eps(FT)), + # ) vertical_transport!( Yₜ.c.ρq_tot[colidx], ᶜJ[colidx], diff --git a/src/prognostic_equations/edmfx_tke.jl b/src/prognostic_equations/edmfx_tke.jl index 477a2a36eee..8268dd70e74 100644 --- a/src/prognostic_equations/edmfx_tke.jl +++ b/src/prognostic_equations/edmfx_tke.jl @@ -28,7 +28,7 @@ function edmfx_tke_tendency!( turbconv_model isa PrognosticEDMFX ? Y.c.sgsʲs.:($j).ρa[colidx] : p.precomputed.ᶜρaʲs.:($j)[colidx] @. ᶜtke_press += - ᶜρaʲ_colidx * + max(ᶜρaʲ_colidx, 0) * adjoint(ᶜinterp.(ᶠu³ʲs.:($$j)[colidx] - ᶠu³⁰[colidx])) * ᶜinterp(C3(nh_pressure3ʲs.:($$j)[colidx])) end @@ -48,7 +48,7 @@ function edmfx_tke_tendency!( turbconv_model isa PrognosticEDMFX ? Y.c.sgsʲs.:($j).ρa[colidx] : p.precomputed.ᶜρaʲs.:($j)[colidx] @. Yₜ.c.sgs⁰.ρatke[colidx] += - ᶜρaʲ_colidx * ( + max(ᶜρaʲ_colidx, 0) * ( ᶜdetrʲs.:($$j)[colidx] * 1 / 2 * norm_sqr( ᶜinterp(ᶠu³⁰[colidx]) - ᶜinterp(ᶠu³ʲs.:($$j)[colidx]), ) - ᶜentrʲs.:($$j)[colidx] * ᶜtke⁰[colidx] diff --git a/src/prognostic_equations/hyperdiffusion.jl b/src/prognostic_equations/hyperdiffusion.jl index cf046c05d9c..e1d7d285bd7 100644 --- a/src/prognostic_equations/hyperdiffusion.jl +++ b/src/prognostic_equations/hyperdiffusion.jl @@ -145,7 +145,7 @@ NVTX.@annotate function apply_hyperdiffusion_tendency!(Yₜ, Y, p, t) # Sub-grid scale hyperdiffusion continued if (turbconv_model isa PrognosticEDMFX) && diffuse_tke - @. Yₜ.c.sgs⁰.ρatke -= ν₄_vorticity * wdivₕ(ᶜρa⁰ * gradₕ(ᶜ∇²tke⁰)) + @. Yₜ.c.sgs⁰.ρatke -= 4 * ν₄_vorticity * wdivₕ(ᶜρa⁰ * gradₕ(ᶜ∇²tke⁰)) end if turbconv_model isa PrognosticEDMFX for j in 1:n diff --git a/toml/longrun_aquaplanet_equil_helem12_clearsky_0M.toml b/toml/longrun_aquaplanet_equil_helem12_clearsky_0M.toml new file mode 100644 index 00000000000..5697ca1f47e --- /dev/null +++ b/toml/longrun_aquaplanet_equil_helem12_clearsky_0M.toml @@ -0,0 +1,11 @@ +[zd_rayleigh] +value = 35000.0 + +[alpha_rayleigh_uh] +value = 0.0 + +[alpha_rayleigh_w] +value = 10.0 + +[precipitation_timescale] +value = 600 diff --git a/toml/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.toml b/toml/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.toml new file mode 100644 index 00000000000..76e5a39ca27 --- /dev/null +++ b/toml/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.toml @@ -0,0 +1,14 @@ +[zd_rayleigh] +value = 35000.0 + +[alpha_rayleigh_uh] +value = 0.0 + +[zd_viscous] +value = 35000.0 + +[kappa_2_sponge] +value = 4e6 + +[precipitation_timescale] +value = 600 diff --git a/toml/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.toml b/toml/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.toml new file mode 100644 index 00000000000..37497a54044 --- /dev/null +++ b/toml/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.toml @@ -0,0 +1,50 @@ +[zd_rayleigh] +value = 35000.0 + +[alpha_rayleigh_uh] +value = 0.0 + +[alpha_rayleigh_w] +value = 10.0 + +[precipitation_timescale] +value = 600 + +[EDMF_surface_area] +value = 0.1 + +[EDMF_min_area] +value = 1.0e-5 + +[EDMF_max_area] +value = 0.7 + +[entr_inv_tau] +value = 0.0002 + +[entr_coeff] +value = 0.2 + +[min_area_limiter_scale] +value = 0 + +[min_area_limiter_power] +value = 100 + +[detr_inv_tau] +value = 0 + +[detr_coeff] +value = 0 + +[detr_buoy_coeff] +value = 0 + +[detr_vertdiv_coeff] +value = 0 + +[detr_massflux_vertdiv_coeff] +value = 1 + +[pressure_normalmode_drag_coeff] +value = 40.0 diff --git a/toml/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.toml b/toml/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.toml index c78bc0e113e..c76bd0e1e89 100644 --- a/toml/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.toml +++ b/toml/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.toml @@ -8,7 +8,7 @@ value = 0.0 value = 600 [EDMF_surface_area] -value = 0 +value = 0.1 [EDMF_min_area] value = 1.0e-5