Skip to content

Commit

Permalink
t push # This is a combination of 14 commits.
Browse files Browse the repository at this point in the history
LES exp ClimaAtmos
Up to 0.14.7 CC

test bugfix
	modified:   src/surface_conditions/surface_conditions.jl

Update manifest for SSPKnoth + mod default

	modified:   config/model_configs/les_dycoms_box.yml
Test pipeline
  • Loading branch information
Akshay Sridhar committed Jun 6, 2024
1 parent 611dd4f commit c47b9d4
Show file tree
Hide file tree
Showing 30 changed files with 503 additions and 2,849 deletions.
184 changes: 12 additions & 172 deletions .buildkite/longruns_gpu/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,14 @@ agents:
modules: julia/1.10.0 cuda/julia-pref openmpi/4.1.5-mpitrampoline nsight-systems/2024.2.1

env:
JULIA_CUDA_MEMORY_POOL: none
JULIA_MPI_HAS_CUDA: "true"
JULIA_NVTX_CALLBACKS: gc
JULIA_MAX_NUM_PRECOMPILE_FILES: 100
OPENBLAS_NUM_THREADS: 1
OMPI_MCA_opal_warn_on_missing_libcuda: 0
SLURM_KILL_BAD_EXIT: 1
SLURM_GPU_BIND: none # https://github.com/open-mpi/ompi/issues/11949#issuecomment-1737712291
CONFIG_PATH: "config/longrun_configs"
SLURM_GRES_FLAGS: "allow-task-sharing"
CONFIG_PATH: "config/model_configs/"
CLIMAATMOS_GC_NSTEPS: 10

timeout_in_minutes: 1440
Expand All @@ -22,6 +21,8 @@ steps:
key: "init_gpu_env"
command:
- echo "--- Instantiate examples"
- "julia --project=examples -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.add(url = \"https://github.com/CLiMA/ClimaTimeSteppers.jl\", rev = \"gb/rosenbrock\");'"
- "julia --project=examples -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.add(url = \"https://github.com/CLiMA/ClimaCore.jl\", rev = \"as/covar-deriv\");'"
- julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)'
- julia --project=examples -e 'using Pkg; Pkg.precompile()'
- julia --project=examples -e 'using CUDA; CUDA.precompile_runtime()'
Expand All @@ -39,178 +40,17 @@ steps:

- wait

- group: "helem 30 dycore"
- group: "test les"
steps:

- label: ":computer: hydrostatic balance (ρe_tot)"
- label: ":computer: dycoms"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
- julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
slurm_gpus_per_task: 1
slurm_cpus_per_task: 4
slurm_ntasks: 4
slurm_exclusive:
env:
JOB_NAME: "longrun_sphere_hydrostatic_balance_rhoe"

- label: ":computer: dry baroclinic wave"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_bw_rhoe_highres"

- label: ":computer: baroclinic wave equilmoist"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_bw_rhoe_equil_highres"

- label: ":computer: dry held-suarez"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_hs_rhoe_dry_55km_nz63"

- label: ":computer: held-suarez, equilmoist"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_hs_rhoe_equil_55km_nz63_0M"

- label: ":computer: held-suarez equilmoist + deep-atmosphere eqns"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
JOB_NAME: "longrun_hs_rhoe_equil_55km_nz63_0M_deepatmos"
agents:
slurm_gpus: 1
slurm_time: 12:00:00

- group: "helem 16 aquaplanet"
steps:

- label: ":computer: aquaplanet equilmoist gray radiation + 0M microphysics"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_gray_0M"
agents:
slurm_gpus: 1
slurm_time: 12:00:00

- label: ":computer: aquaplanet equilmoist clearsky radiation + 0M microphysics"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_0M"

- label: ":computer: aquaplanet equilmoist clearsky radiation + diagnostic edmf + 0M microphysics"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_diagedmf_0M"

- label: ":computer: aquaplanet equilmoist allsky radiation + diagnostic edmf + 0M microphysics"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_allsky_diagedmf_0M"

- label: ":computer: aquaplanet equilmoist clearsky radiation + 0M microphysics + earth topography"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_0M_earth"

- label: ":computer: aquaplanet equilmoist clearsky radiation + 0M microphysics + earth topography (SLEVE)"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 6:00:00
env:
JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_0M_earth_sleve"

- label: ":umbrella: aquaplanet equilmoist clearsky radiation + 1M microphysics"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 6:00:00
env:
JOB_NAME: "longrun_aquaplanet_clearsky_1M"

- label: ":computer: aquaplanet equilmoist clearsky radiation + time-varying insolation + 0M microphysics + slab ocean"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean"

- group: "DYAMOND"

steps:

- label: ":computer: aquaplanet dyamond"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "longrun_aquaplanet_dyamond"

- group: "atmos-only coupler runs"

steps:

- label: ":computer: amip target diagnostic edmf"
command:
- srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
artifact_paths: "$$JOB_NAME/output_active/*"
agents:
slurm_gpus: 1
slurm_time: 12:00:00
env:
JOB_NAME: "amip_target_diagedmf"
JOB_NAME: "les_dycoms_box"
2 changes: 2 additions & 0 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,14 @@ steps:
- "julia --project -e 'using Pkg; Pkg.status()'"

- echo "--- Instantiate examples"
- "julia --project=examples -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.add(url = \"https://github.com/CLiMA/ClimaTimeSteppers.jl\", rev = \"gb/rosenbrock\");'"
- "julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
- "julia --project=examples -e 'using Pkg; Pkg.precompile()'"
- "julia --project=examples -e 'using CUDA; CUDA.precompile_runtime()'"
- "julia --project=examples -e 'using Pkg; Pkg.status()'"

- echo "--- Instantiate perf"
- "julia --project=perf -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.add(url = \"https://github.com/CLiMA/ClimaTimeSteppers.jl\", rev = \"gb/rosenbrock\");'"
- "julia --project=perf -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
- "julia --project=perf -e 'using Pkg; Pkg.precompile()'"
- "julia --project=perf -e 'using Pkg; Pkg.status()'"
Expand Down
9 changes: 8 additions & 1 deletion config/default_configs/default_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ vert_diff:
value: "false"
hyperdiff:
help: "Hyperdiffusion [`ClimaHyperdiffusion` (or `true`; default), `none` (or `false`)]"
value: "ClimaHyperdiffusion"
value: "CAM_SE"
bubble:
help: "Enable bubble correction for more accurate surface areas"
value: true
Expand Down Expand Up @@ -196,6 +196,13 @@ perf_summary:
viscous_sponge:
help: "Viscous sponge [`true`, `false` (default)]"
value: false
smagorinsky_lilly:
value: false
c_smag:
value: 0.2
job_id:
help: "Uniquely identifying string for a particular job"
value: ~
tracer_upwinding:
help: "Tracer upwinding mode [`none` (default), `first_order` , `third_order`, `boris_book`, `zalesak`]"
value: none
Expand Down
20 changes: 11 additions & 9 deletions config/model_configs/box_density_current_test.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,21 @@
job_id: "box_density_current_test"
dt_save_state_to_disk: "10secs"
initial_condition: "DryDensityCurrentProfile"
x_max: 51200.0
z_elem: 45
dt: "0.1secs"
t_end: "10.0secs"
dt: "0.3secs"
t_end: "1000secs"
discrete_hydrostatic_balance: true
y_max: 6400.0
y_elem: 15
hyperdiff: "false"
y_max: 51200.0
y_elem: 45
z_stretch: false
x_elem: 45
ode_algo: "SSP33ShuOsher"
config: "box"
vorticity_hyperdiffusion_coefficient: 0.6
scalar_hyperdiffusion_coefficient: 0.6
hyperdiff: "true"
z_max: 6400.0
smagorinsky_lilly: true
c_smag: 0.25
diagnostics:
- short_name: thetaa
period: 10secs
- short_name: [thetaa, wa, ua, ta, va, hfes, rhoa, rv, ha]
period: 30secs
36 changes: 36 additions & 0 deletions config/model_configs/les_bomex_box.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
job_id: "les_bomex_box"
initial_condition: "Bomex"
edmf_coriolis: "Bomex"
subsidence: "Bomex"
ls_adv: "Bomex"
surface_setup: "Bomex"
moist: "equil"
config: "box"
hyperdiff: "true"
implicit_diffusion: false
precip_model: "1M"
vorticity_hyperdiffusion_coefficient: 0.01
scalar_hyperdiffusion_coefficient: 0.01
smagorinsky_lilly: true
c_smag: 0.17
x_max: 6.4e3
y_max: 6.4e3
z_max: 4.5e3
x_elem: 16
y_elem: 16
z_elem: 75
z_stretch: false
dt: "0.10secs"
output_default_diagnostics: false
t_end: "21600secs"
dt_save_state_to_disk: "30mins"
rayleigh_sponge: true
toml: [toml/bomex_box_rhoe.toml]
restart_file: "./restart/bomex/Micro1M_day0.0_Prod.hdf5"
netcdf_interpolation_num_points: [40, 40, 80]
diagnostics:
- short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl, kvis]
period: 10mins
- short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl, kvis]
reduction: "average"
period: 60mins
36 changes: 36 additions & 0 deletions config/model_configs/les_dycoms_box.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
job_id: "les_dycoms_box"
perturb_initstate: true
initial_condition: DYCOMS_RF01
edmf_coriolis: DYCOMS_RF01
subsidence: DYCOMS
rad: DYCOMS_RF01
surface_setup: DYCOMS_RF01
moist: "equil"
config: box
hyperdiff: "true"
vorticity_hyperdiffusion_coefficient: 0.01
scalar_hyperdiffusion_coefficient: 0.01
ode_algo: "SSPKnoth"
precip_model: "0M"
x_max: 500.0
y_max: 500.0
z_max: 1.5e3
x_elem: 5
y_elem: 5
z_elem: 150
z_stretch: false
smagorinsky_lilly: true
c_smag: 0.20
dt: "0.05secs"
t_end: "14400secs"
dt_save_state_to_disk: "30mins"
restart_file: "/home/asridhar/Codes/ClimaAtmos.jl/restart/dycoms/test_restart.hdf5"
rayleigh_sponge: true
toml: [toml/dycoms_box_rhoe.toml]
netcdf_interpolation_num_points: [30,30,75]
diagnostics:
- short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl, kvis, hfes]
period: 10mins
- short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl, kvis, hfes]
reduction: "average"
period: 60mins
26 changes: 26 additions & 0 deletions config/model_configs/les_gabls_box.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
job_id: "les_gabls_box"
edmf_coriolis: GABLS
initial_condition: GABLS
surface_setup: GABLS
implicit_diffusion: false
moist: "equil"
precip_model: "1M"
config: "box"
hyperdiff: "false"
x_max: 1e3
y_max: 1e3
z_max: 500
x_elem: 5
y_elem: 5
z_elem: 10
z_stretch: false
dt: "0.10secs"
t_end: "100secs"
dt_save_state_to_disk: "30mins"
perturb_initstate: true
netcdf_output_at_levels: true
netcdf_interpolation_num_points: [20, 20, 20]
#restart_file: "restart/gabls/day0.0.hdf5"
diagnostics:
- short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]
period: 10secs
Loading

0 comments on commit c47b9d4

Please sign in to comment.