Skip to content

Commit

Permalink
Merge pull request #1106 from CliMA/gb/outputdir
Browse files Browse the repository at this point in the history
Restructure output directory
  • Loading branch information
Sbozzolo authored Dec 7, 2024
2 parents f66d5df + b599c35 commit 5d528f1
Show file tree
Hide file tree
Showing 11 changed files with 130 additions and 79 deletions.
29 changes: 24 additions & 5 deletions NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,33 @@ As a part of the post processing pipeline, bias plots for variables at the
pressure levels of 850.0, 500.0, 250.0 hPa and bias plots over latitude and
pressure levels are being created.

### Code cleanup

#### Output path updates - PRs [#1106](https://github.com/CliMA/ClimaCoupler.jl/pull/1058),
[#1106](https://github.com/CliMA/ClimaCoupler.jl/pull/1106)

### Code cleanup
#### Output path update - PR [#1058](https://github.com/CliMA/ClimaCoupler.jl/pull/1058)
Previously, ClimaEarth simulation outputs were saved in a path
`experiments/ClimaEarth/output/$mode_name/$job_id/artifacts/`.
This PR removes `mode_name` has from this pattern, so output will now be in
`experiments/ClimaEarth/output/$job_id/artifacts/`.
`experiments/ClimaEarth/output/$mode_name/$job_id/artifacts/`. Now, `ClimaEarth`
creates output folders with an increment (increasing the counter every time the
simulation is run). This is in preparation to restarts. The output now looks
like
```
coupler_output_dir_amip/
├── checkpoints
│ └── checkpoints for the various models
├── artifacts
│ └── plots produced by the postporcessing step
├── output_0000/
│ ├── atmos/
│ │ └── output of the atmos model
│ └── ocean/
│ └── output of the ocean model
├── output_0001/
│ └── ... component model outputs in their folders ...
├── output_0002/
│ └── ... component model outputs in their folders ...
└── output_active -> output_0002/
``
Note that any external scripts that assume an output path will need to be updated.
#### Remove ClimaCoupler.Diagnostics module - PR [#953](https://github.com/CliMA/ClimaCoupler.jl/pull/953)
Expand Down
2 changes: 2 additions & 0 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ version = "0.1.1"
ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d"
ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884"
ClimaCoreTempestRemap = "d934ef94-cdd4-4710-83d6-720549644b70"
ClimaUtilities = "b3f4f4ca-9299-4f7f-bd9b-81e1242a7513"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
Expand All @@ -20,6 +21,7 @@ Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c"
ClimaComms = "0.5.6, 0.6"
ClimaCore = "0.14.19"
ClimaCoreTempestRemap = "0.3"
ClimaUtilities = "0.1.14"
Dates = "1"
JLD2 = "0.4, 0.5"
Logging = "1"
Expand Down
6 changes: 3 additions & 3 deletions experiments/ClimaEarth/components/atmosphere/climaatmos.jl
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ FluxCalculator.get_surface_params(sim::ClimaAtmosSimulation) = CAP.surface_fluxe
### ClimaAtmos.jl model-specific functions (not explicitly required by ClimaCoupler.jl)
###
"""
get_atmos_config_dict(coupler_dict::Dict, job_id::String)
get_atmos_config_dict(coupler_dict::Dict, job_id::String, atmos_output_dir)
Returns the specified atmospheric configuration (`atmos_config`) overwitten by arguments
in the coupler dictionary (`config_dict`).
Expand All @@ -313,7 +313,7 @@ The TOML parameter file to use is chosen using the following priority:
If a coupler TOML file is provided, it is used. Otherwise we use an atmos TOML
file if it's provided. If neither is provided, we use a default coupler TOML file.
"""
function get_atmos_config_dict(coupler_dict::Dict, job_id::String)
function get_atmos_config_dict(coupler_dict::Dict, job_id::String, atmos_output_dir)
atmos_config_file = coupler_dict["atmos_config_file"]
atmos_config_repo = coupler_dict["atmos_config_repo"]
# override default or specified configs with coupler arguments, and set the correct atmos config_file
Expand Down Expand Up @@ -357,7 +357,7 @@ function get_atmos_config_dict(coupler_dict::Dict, job_id::String)
end

# Specify atmos output directory to be inside the coupler output directory
atmos_output_dir = joinpath(coupler_dict["coupler_output_dir"], job_id, "clima_atmos")
atmos_config["output_dir_style"] = "RemovePreexisting"
atmos_config["output_dir"] = atmos_output_dir

# Access extra atmosphere diagnostics from coupler so we can rename for atmos code
Expand Down
2 changes: 1 addition & 1 deletion experiments/ClimaEarth/hierarchy/climate_plots.jl
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ for job_id in ["dry_held_suarez", "moist_held_suarez"]
DATA_DIR = "experiments/ClimaEarth/$job_id/$job_id/clima_atmos/output_active/"
else
build = ENV["BUILDKITE_BUILD_NUMBER"]
DATA_DIR = "/central/scratch/esm/slurm-buildkite/climacoupler-ci/$build/climacoupler-ci/$job_id/$job_id/clima_atmos/output_active/"
DATA_DIR = "/central/scratch/esm/slurm-buildkite/climacoupler-ci/$build/climacoupler-ci/$job_id/output_active/clima_atmos/"
end

reduction = "6h_inst"
Expand Down
39 changes: 20 additions & 19 deletions experiments/ClimaEarth/run_amip.jl
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,25 @@ add_extra_diagnostics!(config_dict)
plot_diagnostics,
) = get_coupler_args(config_dict)

#=
### I/O Directory Setup `setup_output_dirs` returns `dir_paths.output =
COUPLER_OUTPUT_DIR`, which is the directory where the output of the simulation
will be saved, `dir_paths.artifacts` is the directory where the plots (from
postprocessing and the conservation checks) of the simulation will be saved,
#and `dir_paths.checkpoints`, where restart files are saved.
=#

COUPLER_OUTPUT_DIR = joinpath(output_dir_root, job_id)
dir_paths = Utilities.setup_output_dirs(output_dir = COUPLER_OUTPUT_DIR, comms_ctx = comms_ctx)
@info "Coupler output directory $(dir_paths.output)"
@info "Coupler artifacts directory $(dir_paths.artifacts)"
@info "Coupler checkpoint directory $(dir_paths.checkpoints)"

## get component model dictionaries (if applicable)
## Note this step must come after parsing the coupler config dictionary, since
## some parameters are passed from the coupler config to the component model configs
atmos_config_dict = get_atmos_config_dict(config_dict, job_id)
atmos_output_dir = joinpath(dir_paths.output, "clima_atmos")
atmos_config_dict = get_atmos_config_dict(config_dict, job_id, atmos_output_dir)
(; dt_rad, output_default_diagnostics) = get_atmos_args(atmos_config_dict)

## set unique random seed if desired, otherwise use default
Expand All @@ -137,18 +152,6 @@ Random.seed!(random_seed)

tspan = (t_start, t_end)

#=
### I/O Directory Setup
`Utilities.setup_output_dirs` returns `dir_paths.output = COUPLER_OUTPUT_DIR`, which is the directory where the output of the simulation will be saved, and `dir_paths.artifacts` is the directory where
the plots (from postprocessing and the conservation checks) of the simulation will be saved. `dir_paths.regrid` is the directory where the regridding
temporary files will be saved.
=#

COUPLER_OUTPUT_DIR = joinpath(output_dir_root, job_id)
dir_paths = Utilities.setup_output_dirs(output_dir = COUPLER_OUTPUT_DIR, comms_ctx = comms_ctx)
@info "Coupler output directory $(dir_paths.output)"
@info "Coupler artifacts directory $(dir_paths.artifacts)"

#=
## Data File Paths
=#
Expand Down Expand Up @@ -848,13 +851,12 @@ if ClimaComms.iamroot(comms_ctx)

# define variable names and output directories for each diagnostic
amip_short_names_atmos = ["ta", "ua", "hus", "clw", "pr", "ts", "toa_fluxes_net"]
output_dir_atmos = atmos_sim.integrator.p.output_dir
amip_short_names_coupler = ["F_turb_energy"]
output_dir_coupler = dir_paths.output

# Check if all output variables are available in the specified directories
make_diagnostics_plots(
output_dir_atmos,
atmos_output_dir,
dir_paths.artifacts,
short_names = amip_short_names_atmos,
output_prefix = "atmos_",
Expand All @@ -870,17 +872,16 @@ if ClimaComms.iamroot(comms_ctx)
# Check this because we only want monthly data for making plots
if t_end > 84600 * 31 * 3 && output_default_diagnostics
include("leaderboard/leaderboard.jl")
diagnostics_folder_path = atmos_sim.integrator.p.output_dir
leaderboard_base_path = dir_paths.artifacts
compute_leaderboard(leaderboard_base_path, diagnostics_folder_path)
compute_pfull_leaderboard(leaderboard_base_path, diagnostics_folder_path)
compute_leaderboard(leaderboard_base_path, atmos_output_dir)
compute_pfull_leaderboard(leaderboard_base_path, atmos_output_dir)
end
end
## plot extra atmosphere diagnostics if specified
if plot_diagnostics
@info "Plotting diagnostics"
include("user_io/diagnostics_plots.jl")
make_diagnostics_plots(atmos_sim.integrator.p.output_dir, dir_paths.artifacts)
make_diagnostics_plots(atmos_output_dir, dir_paths.artifacts)
end

## plot all model states and coupler fields (useful for debugging)
Expand Down
17 changes: 9 additions & 8 deletions experiments/ClimaEarth/run_cloudless_aquaplanet.jl
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ start_date = "19790301"
hourly_checkpoint = true
dt_rad = "6hours"

#=
### I/O Directory Setup
=#

dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = ClimaComms.context())
@info(config_dict)

## namelist
config_dict = Dict(
# general
Expand Down Expand Up @@ -109,7 +116,8 @@ config_dict = Dict(
)

## merge dictionaries of command line arguments, coupler dictionary and component model dictionaries
atmos_config_dict = get_atmos_config_dict(config_dict, job_id)
atmos_output_dir = joinpath(dir_paths.output, "clima_atmos")
atmos_config_dict = get_atmos_config_dict(config_dict, job_id, atmos_output_dir)
atmos_config_object = CA.AtmosConfig(atmos_config_dict)

#=
Expand All @@ -118,13 +126,6 @@ atmos_config_object = CA.AtmosConfig(atmos_config_dict)

comms_ctx = Utilities.get_comms_context(Dict("device" => "auto"))

#=
### I/O Directory Setup
=#

dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = comms_ctx)
@info(config_dict)

#=
## Component Model Initialization
=#
Expand Down
17 changes: 9 additions & 8 deletions experiments/ClimaEarth/run_cloudy_aquaplanet.jl
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ start_date = "19790301"
hourly_checkpoint = true
dt_rad = "6hours"

#=
### I/O Directory Setup
=#

dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = ClimaComms.context())
@info(config_dict)

## namelist
config_dict = Dict(
# general
Expand Down Expand Up @@ -122,7 +129,8 @@ config_dict = Dict(
)

## merge dictionaries of command line arguments, coupler dictionary and component model dictionaries
atmos_config_dict = get_atmos_config_dict(config_dict, job_id)
atmos_output_dir = joinpath(dir_paths.output, "clima_atmos")
atmos_config_dict = get_atmos_config_dict(config_dict, job_id, atmos_output_dir)
atmos_config_object = CA.AtmosConfig(atmos_config_dict)

## override default toml parameters
Expand All @@ -143,13 +151,6 @@ comms_ctx = Utilities.get_comms_context(Dict("device" => "auto"))
ClimaComms.init(comms_ctx)


#=
### I/O Directory Setup
=#

dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = comms_ctx)
@info(config_dict)

#=
## Component Model Initialization
=#
Expand Down
18 changes: 10 additions & 8 deletions experiments/ClimaEarth/run_cloudy_slabplanet.jl
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,14 @@ start_date = "19790321"
hourly_checkpoint = true
dt_rad = "6hours"

#=
### I/O Directory Setup
=#

dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = ClimaComms.context())
@info(config_dict)


## namelist
config_dict = Dict(
# general
Expand Down Expand Up @@ -126,7 +134,8 @@ config_dict = Dict(
)

## merge dictionaries of command line arguments, coupler dictionary and component model dictionaries
atmos_config_dict = get_atmos_config_dict(config_dict, job_id)
atmos_output_dir = joinpath(dir_paths.output, "clima_atmos")
atmos_config_dict = get_atmos_config_dict(config_dict, job_id, atmos_output_dir)
atmos_config_object = CA.AtmosConfig(atmos_config_dict)

# override default toml parameters
Expand All @@ -145,13 +154,6 @@ atmos_config_object.toml_dict["max_area_limiter_scale"]["value"] = 0

comms_ctx = Utilities.get_comms_context(Dict("device" => "auto"))

#=
### I/O Directory Setup
=#

dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = comms_ctx)
@info(config_dict)

#=
## Data File Paths
=#
Expand Down
17 changes: 9 additions & 8 deletions experiments/ClimaEarth/run_dry_held_suarez.jl
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ tspan = (Float64(0.0), Float64(Utilities.time_to_seconds(t_end)))
start_date = "19790301"
hourly_checkpoint = true

#=
### I/O Directory Setup
=#

dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = ClimaComms.context())
@info(config_dict)

## namelist
config_dict = Dict(
# general
Expand Down Expand Up @@ -101,7 +108,8 @@ config_dict = Dict(
)

## merge dictionaries of command line arguments, coupler dictionary and component model dictionaries
atmos_config_dict = get_atmos_config_dict(config_dict, job_id)
atmos_output_dir = joinpath(dir_paths.output, "clima_atmos")
atmos_config_dict = get_atmos_config_dict(config_dict, job_id, atmos_output_dir)
atmos_config_object = CA.AtmosConfig(atmos_config_dict)

#=
Expand All @@ -110,13 +118,6 @@ atmos_config_object = CA.AtmosConfig(atmos_config_dict)

comms_ctx = Utilities.get_comms_context(Dict("device" => "auto"))

#=
### I/O Directory Setup
=#

dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = comms_ctx)
@info(config_dict)

#=
## Component Model Initialization
=#
Expand Down
18 changes: 10 additions & 8 deletions experiments/ClimaEarth/run_moist_held_suarez.jl
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,14 @@ tspan = (Float64(0.0), Float64(Utilities.time_to_seconds(t_end)))
start_date = "19790301"
hourly_checkpoint = true

#=
### I/O Directory Setup
=#


dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = ClimaComms.context())
@info(config_dict)

## namelist
config_dict = Dict(
# general
Expand Down Expand Up @@ -109,7 +117,8 @@ config_dict = Dict(
# TODO: may need to switch to Bulk fluxes

## merge dictionaries of command line arguments, coupler dictionary and component model dictionaries
atmos_config_dict = get_atmos_config_dict(config_dict, job_id)
atmos_output_dir = joinpath(dir_paths.output, "clima_atmos")
atmos_config_dict = get_atmos_config_dict(config_dict, job_id, atmos_output_dir)
atmos_config_object = CA.AtmosConfig(atmos_config_dict)

#=
Expand All @@ -118,13 +127,6 @@ atmos_config_object = CA.AtmosConfig(atmos_config_dict)

comms_ctx = Utilities.get_comms_context(Dict("device" => "auto"))

#=
### I/O Directory Setup
=#

dir_paths = Utilities.setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = comms_ctx)
@info(config_dict)

#=
## Component Model Initialization
=#
Expand Down
Loading

0 comments on commit 5d528f1

Please sign in to comment.