From 3770df6a44bf082d30421315910dda0c602a6711 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 20 Jul 2022 15:18:41 -0600 Subject: [PATCH 1/4] Add ability to split simulations into partitions --- Project.toml | 1 + docs/src/man/parallel_simulations_hpc.md | 196 ++++++++++++ docs/src/man/parallel_simulations_local.md | 80 +++++ src/PowerSimulations.jl | 9 + src/core/definitions.jl | 1 + src/simulation/simulation.jl | 90 +++++- src/simulation/simulation_internal.jl | 58 ++-- .../simulation_partition_results.jl | 182 +++++++++++ src/simulation/simulation_partitions.jl | 282 ++++++++++++++++++ src/simulation/simulation_results.jl | 5 +- test/run_partitioned_simulation.jl | 171 +++++++++++ test/test_simulation_partitions.jl | 109 +++++++ 12 files changed, 1135 insertions(+), 49 deletions(-) create mode 100644 docs/src/man/parallel_simulations_hpc.md create mode 100644 docs/src/man/parallel_simulations_local.md create mode 100644 src/simulation/simulation_partition_results.jl create mode 100644 src/simulation/simulation_partitions.jl create mode 100644 test/run_partitioned_simulation.jl create mode 100644 test/test_simulation_partitions.jl diff --git a/Project.toml b/Project.toml index ece24537d0..9af44fd05c 100644 --- a/Project.toml +++ b/Project.toml @@ -8,6 +8,7 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" InfrastructureSystems = "2cd47ed4-ca9b-11e9-27f2-ab636a7671f1" diff --git a/docs/src/man/parallel_simulations_hpc.md b/docs/src/man/parallel_simulations_hpc.md new file mode 100644 index 0000000000..bde9c8a72c --- /dev/null +++ b/docs/src/man/parallel_simulations_hpc.md @@ -0,0 +1,196 @@ +## Run a Simulation in Parallel on an HPC + +This page describes how to split a simulation into partitions, run each partition in parallel +on HPC compute nodes, and then join the results. + +These steps can be used on a local computer or any HPC supported by the submission software. +Some steps may be specific to NREL's HPC `Eagle` cluster. + +*Note*: Some instructions are preliminary and will change if functionality is moved +to a new Julia package. + +### Setup + +1. Create a conda environment and install the Python package `NREL-jade`: + https://nrel.github.io/jade/installation.html. The rest of this page assumes that + the environment is called `jade`. +2. Activate the environment with `conda activate jade`. +3. Locate the path to that conda environment. It will likely be `~/.conda-envs/jade` or + `~/.conda/envs/jade`. +4. Load the Julia environment that you use to run simulations. Add the packages `Conda` and + `PyCall`. +5. Setup Conda to use the existing `jade` environment by running these commands: + +``` +julia> run(`conda create -n conda_jl python conda`) +julia> ENV["CONDA_JL_HOME"] = joinpath(ENV["HOME"], ".conda-envs", "jade") # change this to your path +pkg> build Conda +``` + +6. Copy the code below into a Julia file called `configure_parallel_simulation.jl`. + This is an interface to Jade through PyCall. It will be used to create a Jade configuration. + (It may eventually be moved to a separate package.) + +``` +function configure_parallel_simulation( + script::AbstractString, + num_steps::Integer, + num_period_steps::Integer; + num_overlap_steps::Integer=0, + project_path=nothing, + simulation_name="simulation", + config_file="config.json", + force=false, +) + partitions = SimulationPartitions(num_steps, num_period_steps, num_overlap_steps) + jgc = pyimport("jade.extensions.generic_command") + julia_cmd = isnothing(project_path) ? "julia" : "julia --project=$project_path" + setup_command = "$julia_cmd $script setup --simulation-name=$simulation_name " * + "--num-steps=$num_steps --num-period-steps=$num_period_steps " * + "--num-overlap-steps=$num_overlap_steps" + teardown_command = "$julia_cmd $script join --simulation-name=$simulation_name" + config = jgc.GenericCommandConfiguration( + setup_command=setup_command, + teardown_command=teardown_command, + ) + + for i in 1:get_num_partitions(partitions) + cmd = "$julia_cmd $script execute --simulation-name=$simulation_name --index=$i" + job = jgc.GenericCommandParameters(command=cmd, name="execute-$i") + config.add_job(job) + end + + config.dump(config_file, indent=2) + println("Created Jade configuration in $config_file. " * + "Run 'jade submit-jobs [options] $config_file' to execute them.") +end +``` + +7. Create a Julia script to build and run simulations. It must meet the requirements below. + A full example is in the PowerSimulations repository in `test/run_partitioned_simulation.jl`. + +- Call `using PowerSimulations`. + +- Implement a build function that matches the signature below. + It must construct a `Simulation`, call `build!`, and then return the `Simulation` instance. + It must throw an exception if the build fails. + +``` +function build_simulation( + output_dir::AbstractString, + simulation_name::AbstractString, + partitions::SimulationPartitions, + index::Union{Nothing, Integer}=nothing, +) +``` + +Here is example code to construct the `Simulation` with these parameters: + +``` + sim = Simulation( + name=simulation_name, + steps=partitions.num_steps, + models=models, + sequence=sequence, + simulation_folder=output_dir, + ) + status = build!(sim; partitions=partitions, index=index, serialize=isnothing(index)) + if status != PSI.BuildStatus.BUILT + error("Failed to build simulation: status=$status") + end +``` + +- Implement an execute function that matches the signature below. It must throw an exception + if the execute fails. + +``` +function execute_simulation(sim, args...; kwargs...) + status = execute!(sim) + if status != PSI.RunStatus.SUCCESSFUL + error("Simulation failed to execute: status=$status") + end +end +``` + +- Make the script runnable as a CLI command by including the following code at the bottom of the +file. + +``` +function main() + process_simulation_partition_cli_args(build_simulation, execute_simulation, ARGS...) +end + +if abspath(PROGRAM_FILE) == @__FILE__ + main() +end +``` + +### Execution + +1. Create a Jade configuration that defines the partitioned simulation jobs. Load your Julia + environment. + + This example splits a year-long simulation into weekly partitions for a total of 53 individual + jobs. + +``` +julia> include("configure_parallel_simulation.jl") +julia> num_steps = 365 +julia> period = 7 +julia> num_overlap_steps = 1 +julia> configure_parallel_simulation( + "my_simulation.jl", # this is your build/execute script + num_steps, + period, + num_overlap_steps=1, + project_path=".", # This optionally specifies the Julia project environment to load. +) +Created Jade configuration in config.json. Run 'jade submit-jobs [options] config.json' to execute them. +``` + +Exit Julia. + +2. View the configuration for accuracy. + +``` +$ jade config show config.json +``` + +3. Start an interactive session on a debug node. *Do not submit the jobs on a login node!* The submission + step will run a full build of the simulation and that may consume too many CPU and memory resources + for the login node. + +``` +$ salloc -t 01:00:00 -N1 --account= --partition=debug +``` + +4. Follow the instructions at https://nrel.github.io/jade/tutorial.html to submit the jobs. + The example below will configure Jade to run each partition on its own compute node. Depending on + the compute and memory constraints of your simulation, you may be able to pack more jobs on each + node. + + Adjust the walltime as necessary. + +``` +$ jade config hpc -c hpc_config.toml -t slurm --walltime=04:00:00 -a +$ jade submit-jobs config.json --per-node-batch-size=1 -o output +``` + +If you are unsure about how much memory and CPU resources your simulation consumes, add these options: + +``` +$ jade submit-jobs config.json --per-node-batch-size=1 -o output --resource-monitor-type periodic --resource-monitor-interval 3 +``` + +Jade will create HTML plots of the resource utilization in `output/stats`. You may be able to customize +`--per-node-batch-size` and `--num-processes` to finish the simulations more quickly. + +5. Jade will run a final command to join the simulation partitions into one unified file. You can load the + results as you normally would. + +``` +julia> results = SimulationResults("/job-outputs/") +``` + +Note the log files and results for each partition are located in +`/job-outputs//simulation_partitions` diff --git a/docs/src/man/parallel_simulations_local.md b/docs/src/man/parallel_simulations_local.md new file mode 100644 index 0000000000..93204fcce2 --- /dev/null +++ b/docs/src/man/parallel_simulations_local.md @@ -0,0 +1,80 @@ +## Run a Simulation in Parallel on a local computer + +This page describes how to split a simulation into partitions, run each partition in parallel, +and then join the results. + +### Setup + +Create a Julia script to build and run simulations. It must meet the requirements below. +A full example is in the PowerSimulations repository in `test/run_partitioned_simulation.jl`. + +- Call `using PowerSimulations`. + +- Implement a build function that matches the signature below. + It must construct a `Simulation`, call `build!`, and then return the `Simulation` instance. + It must throw an exception if the build fails. + +``` +function build_simulation( + output_dir::AbstractString, + simulation_name::AbstractString, + partitions::SimulationPartitions, + index::Union{Nothing, Integer}=nothing, +) +``` + +Here is example code to construct the `Simulation` with these parameters: + +``` + sim = Simulation( + name=simulation_name, + steps=partitions.num_steps, + models=models, + sequence=sequence, + simulation_folder=output_dir, + ) + status = build!(sim; partitions=partitions, index=index, serialize=isnothing(index)) + if status != PSI.BuildStatus.BUILT + error("Failed to build simulation: status=$status") + end +``` + +- Implement an execute function that matches the signature below. It must throw an exception + if the execute fails. + +``` +function execute_simulation(sim, args...; kwargs...) + status = execute!(sim) + if status != PSI.RunStatus.SUCCESSFUL + error("Simulation failed to execute: status=$status") + end +end +``` + +### Execution + +After loading your script, call the function `run_parallel_simulation` as shown below. + +This example splits a year-long simulation into weekly partitions for a total of 53 individual +jobs and then runs them four at a time. + +``` +julia> include("my_simulation.jl") +julia> run_parallel_simulation( + build_simulation, + execute_simulation, + script="my_simulation.jl", + output_dir="my_simulation_output", + name="my_simulation", + num_steps=365, + period=7, + num_overlap_steps=1, + num_parallel_processes=4, + exeflags="--project=", + ) +``` + +The final results will be in `./my_simulation_otuput/my_simulation` + +Note the log files and results for each partition are located in +`./my_simulation_otuput/my_simulation/simulation_partitions` diff --git a/src/PowerSimulations.jl b/src/PowerSimulations.jl index 406cc50203..9b519f7c76 100644 --- a/src/PowerSimulations.jl +++ b/src/PowerSimulations.jl @@ -14,6 +14,8 @@ export InitialCondition export SimulationModels export SimulationSequence export SimulationResults +export SimulationPartitions +export SimulationPartitionResults # Network Relevant Exports export NetworkModel @@ -120,6 +122,7 @@ export run! ## Sim Model Exports export execute! export get_simulation_model +export run_parallel_simulation ## Template Exports export template_economic_dispatch export template_unit_commitment @@ -199,6 +202,7 @@ export show_recorder_events export list_simulation_events export show_simulation_events export export_realized_results +export get_num_partitions ## Enums export BuildStatus @@ -377,6 +381,7 @@ export get_resolution import PowerModels import TimerOutputs import ProgressMeter +import Distributed # Base Imports import Base.getindex @@ -408,6 +413,8 @@ export SOCWRConicPowerModel export QCRMPowerModel export QCLSPowerModel +export process_simulation_partition_cli_args + ################################################################################ # Type Alias From other Packages @@ -508,6 +515,8 @@ include("simulation/simulation_problem_results.jl") include("simulation/realized_meta.jl") include("simulation/decision_model_simulation_results.jl") include("simulation/emulation_model_simulation_results.jl") +include("simulation/simulation_partitions.jl") +include("simulation/simulation_partition_results.jl") include("simulation/simulation_sequence.jl") include("simulation/simulation_internal.jl") include("simulation/simulation.jl") diff --git a/src/core/definitions.jl b/src/core/definitions.jl index d9a86b2689..c27351fe9d 100644 --- a/src/core/definitions.jl +++ b/src/core/definitions.jl @@ -71,6 +71,7 @@ const KNOWN_SIMULATION_PATHS = [ "recorder", "results", "simulation_files", + "simulation_partitions", ] const RESULTS_DIR = "results" diff --git a/src/simulation/simulation.jl b/src/simulation/simulation.jl index 3c787fc564..1750014b24 100644 --- a/src/simulation/simulation.jl +++ b/src/simulation/simulation.jl @@ -22,7 +22,7 @@ mutable struct Simulation name::String, steps::Int, models::SimulationModels, - simulation_folder::String, + simulation_folder::AbstractString, initial_time=nothing, ) for model in get_decision_models(models) @@ -85,6 +85,8 @@ get_simulation_model(s::Simulation, name) = get_simulation_model(get_models(s), get_models(sim::Simulation) = sim.models get_simulation_dir(sim::Simulation) = dirname(sim.internal.logs_dir) get_simulation_files_dir(sim::Simulation) = sim.internal.sim_files_dir +get_simulation_partitions_dir(sim::Simulation) = + joinpath(get_simulation_dir(sim), "simulation_partitions") get_store_dir(sim::Simulation) = sim.internal.store_dir get_simulation_status(sim::Simulation) = sim.internal.status get_simulation_build_status(sim::Simulation) = sim.internal.build_status @@ -431,12 +433,27 @@ function _initialize_problem_storage!( return simulation_store_params end -function _build!(sim::Simulation, serialize::Bool) +function _build!( + sim::Simulation; + serialize=true, + setup_simulation_partitions=false, + partitions=nothing, + index=nothing, +) set_simulation_build_status!(sim, BuildStatus.IN_PROGRESS) problem_initial_times = _get_simulation_initial_times!(sim) sequence = get_sequence(sim) step_resolution = get_step_resolution(sequence) simulation_models = get_models(sim) + + if !isnothing(partitions) && !isnothing(index) + step_range = get_absolute_step_range(partitions, index) + sim.initial_time += step_resolution * (step_range.start - 1) + set_current_time!(sim.internal.simulation_state, sim.initial_time) + sim.steps = length(step_range) + @info "Set parameters for simulation partition" index sim.initial_time sim.steps + end + for (ix, model) in enumerate(get_decision_models(simulation_models)) problem_interval = get_interval(sequence, model) # Note to devs: Here we are setting the number of operations problem executions we @@ -485,12 +502,19 @@ function _build!(sim::Simulation, serialize::Bool) serialize_problem(em) end end + + if setup_simulation_partitions + TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Setup Simulation Partition" begin + _setup_simulation_partitions(sim) + end + end + return end function _set_simulation_internal!( sim::Simulation, - output_dir, + partitions::Union{Nothing, SimulationPartitions}, recorders, console_level, file_level, @@ -500,14 +524,33 @@ function _set_simulation_internal!( get_models(sim), get_simulation_folder(sim), get_name(sim), - output_dir, recorders, console_level, file_level, + partitions=partitions, ) return end +function _setup_simulation_partitions(sim::Simulation) + mkdir(sim.internal.partitions_dir) + filename = joinpath(sim.internal.partitions_dir, "config.json") + IS.to_json(sim.internal.partitions, filename, pretty=true) + for i in 1:get_num_partitions(sim.internal.partitions) + mkdir(joinpath(sim.internal.partitions_dir, string(i))) + end + + open_store(HdfSimulationStore, get_store_dir(sim), "w") do store + set_simulation_store!(sim, store) + _initialize_problem_storage!( + sim, + DEFAULT_SIMULATION_STORE_CACHE_SIZE_MiB, + MIN_CACHE_FLUSH_SIZE_MiB, + ) + end + set_simulation_store!(sim, nothing) +end + """ build!(sim::Simulation) @@ -516,8 +559,6 @@ Build the Simulation, problems and the related folder structure # Arguments - `sim::Simulation`: simulation object - - `output_dir` = nothing: Name of the output directory for the simulation. If nothing, the - folder will have the same name as the simulation - `serialize::Bool = true`: serializes the simulation objects in the simulation - `recorders::Vector{Symbol} = []`: recorder names to register - `console_level = Logging.Error`: @@ -527,18 +568,28 @@ Throws an exception if name is passed and the directory already exists. """ function build!( sim::Simulation; - output_dir=nothing, recorders=[], console_level=Logging.Error, file_level=Logging.Info, serialize=true, - initialize_problem=false, + partitions::Union{Nothing, SimulationPartitions}=nothing, + index=nothing, ) + if !isnothing(partitions) && !isnothing(index) && serialize + # This is build of a partition. No need to serialize again. + serialize = false + end TimerOutputs.reset_timer!(BUILD_PROBLEMS_TIMER) TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Build Simulation" begin - TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Initialize Simulation Internal" begin - _check_folder(sim) - _set_simulation_internal!(sim, output_dir, recorders, console_level, file_level) + _check_folder(sim) + _set_simulation_internal!(sim, partitions, recorders, console_level, file_level) + make_dirs(sim.internal) + if !isnothing(partitions) && isnothing(index) + # This is the build for the overall simulation. + setup_simulation_partitions = true + serialize = true + else + setup_simulation_partitions = false end file_mode = "w" logger = configure_logging(sim.internal, file_mode) @@ -546,7 +597,13 @@ function build!( try Logging.with_logger(logger) do try - _build!(sim, serialize) + _build!( + sim, + serialize=serialize, + setup_simulation_partitions=setup_simulation_partitions, + partitions=partitions, + index=index, + ) set_simulation_build_status!(sim, BuildStatus.BUILT) set_simulation_status!(sim, RunStatus.READY) catch e @@ -561,7 +618,6 @@ function build!( close(logger) end end - initialize_problem && _initial_conditions_problems!(sim) @info "\n$(BUILD_PROBLEMS_TIMER)\n" return get_simulation_build_status(sim) end @@ -1018,8 +1074,12 @@ function deserialize_model( end function serialize_status(sim::Simulation) - data = Dict("run_status" => string(get_simulation_status(sim))) - filename = joinpath(get_results_dir(sim), "status.json") + serialize_status(get_simulation_status(sim), get_results_dir(sim)) +end + +function serialize_status(status::RunStatus, results_dir::AbstractString) + data = Dict("run_status" => string(status)) + filename = joinpath(results_dir, "status.json") open(filename, "w") do io JSON3.write(io, data) end diff --git a/src/simulation/simulation_internal.jl b/src/simulation/simulation_internal.jl index 7c163d0967..e3137a5f0d 100644 --- a/src/simulation/simulation_internal.jl +++ b/src/simulation/simulation_internal.jl @@ -1,10 +1,12 @@ mutable struct SimulationInternal sim_files_dir::String + partitions::Union{Nothing, SimulationPartitions} store_dir::String logs_dir::String models_dir::String recorder_dir::String results_dir::String + partitions_dir::String run_count::OrderedDict{Int, OrderedDict{Int, Int}} date_ref::OrderedDict{Int, Dates.DateTime} status::RunStatus @@ -21,12 +23,12 @@ end function SimulationInternal( steps::Int, models::SimulationModels, - sim_dir::String, + base_dir::String, name::String, - output_dir::Union{Nothing, String}, recorders, console_level::Logging.LogLevel, - file_level::Logging.LogLevel, + file_level::Logging.LogLevel; + partitions::Union{Nothing, SimulationPartitions}=nothing, cache_size_mib=1024, min_cache_flush_size_mib=MIN_CACHE_FLUSH_SIZE_MiB, ) @@ -43,13 +45,9 @@ function SimulationInternal( end end - base_dir = joinpath(sim_dir, name) - mkpath(base_dir) - - output_dir = _get_output_dir_name(base_dir, output_dir) - simulation_dir = joinpath(base_dir, output_dir) + simulation_dir = joinpath(base_dir, name) if isdir(simulation_dir) - error("$simulation_dir already exists. Delete it or pass a different output_dir.") + simulation_dir = _get_output_dir_name(base_dir, name) end sim_files_dir = joinpath(simulation_dir, "simulation_files") @@ -58,29 +56,20 @@ function SimulationInternal( models_dir = joinpath(simulation_dir, "problems") recorder_dir = joinpath(simulation_dir, "recorder") results_dir = joinpath(simulation_dir, RESULTS_DIR) - - for path in ( - simulation_dir, - sim_files_dir, - logs_dir, - models_dir, - recorder_dir, - results_dir, - store_dir, - ) - mkpath(path) - end + partitions_dir = joinpath(simulation_dir, "simulation_partitions") unique_recorders = Set(REQUIRED_RECORDERS) foreach(x -> push!(unique_recorders, x), recorders) return SimulationInternal( sim_files_dir, + partitions, store_dir, logs_dir, models_dir, recorder_dir, results_dir, + partitions_dir, count_dict, Dict{Int, Dates.DateTime}(), RunStatus.NOT_READY, @@ -95,24 +84,29 @@ function SimulationInternal( ) end -function _get_output_dir_name(path, output_dir) - if output_dir !== nothing - # The user wants a custom name. - return output_dir +function make_dirs(internal::SimulationInternal) + mkdir(dirname(internal.sim_files_dir)) + for field in + (:sim_files_dir, :store_dir, :logs_dir, :models_dir, :recorder_dir, :results_dir) + mkdir(getproperty(internal, field)) end +end +function _get_output_dir_name(path, sim_name) # Return the next highest integer. - output_dir = 1 - for name in readdir(path) - if occursin(r"^\d+$", name) - num = parse(Int, name) - if num >= output_dir - output_dir = num + 1 + index = 2 + for path_name in readdir(path) + regex = Regex("\\Q$sim_name\\E-(\\d+)\$") + m = match(regex, path_name) + if !isnothing(m) + num = parse(Int, m.captures[1]) + if num >= index + index = num + 1 end end end - return string(output_dir) + return joinpath(path, "$sim_name-$index") end function configure_logging(internal::SimulationInternal, file_mode) diff --git a/src/simulation/simulation_partition_results.jl b/src/simulation/simulation_partition_results.jl new file mode 100644 index 0000000000..325d478132 --- /dev/null +++ b/src/simulation/simulation_partition_results.jl @@ -0,0 +1,182 @@ +const _TEMP_WRITE_POSITION = "__write_position__" + +""" +Handles merging of simulation partitions +""" +struct SimulationPartitionResults + "Directory of main simulation" + path::String + "User-defined simulation name" + simulation_name::String + "Defines how the simulation is split into partitions" + partitions::SimulationPartitions + "Cache of datasets" + datasets::Dict{String, HDF5.Dataset} +end + +function SimulationPartitionResults(path::AbstractString) + config_file = joinpath(path, "simulation_partitions", "config.json") + config = open(config_file, "r") do io + JSON3.read(io, Dict) + end + partitions = IS.deserialize(SimulationPartitions, config) + return SimulationPartitionResults( + path, + basename(path), + partitions, + Dict{String, HDF5.Dataset}(), + ) +end + +""" +Combine all partition simulation files. +""" +function join_simulation(path::AbstractString) + results = SimulationPartitionResults(path) + join_simulation(results) + return +end + +function join_simulation(results::SimulationPartitionResults) + status = _check_jobs(results) + _merge_store_files!(results) + _complete(results, status) + return +end + +_partition_path(x::SimulationPartitionResults, i) = + joinpath(x.path, "simulation_partitions", string(i), x.simulation_name) +_store_subpath() = joinpath("data_store", "simulation_store.h5") +_store_path(x::SimulationPartitionResults) = joinpath(x.path, _store_subpath()) + +function _check_jobs(results::SimulationPartitionResults) + overall_status = RunStatus.SUCCESSFUL + for i in 1:get_num_partitions(results.partitions) + job_results_path = joinpath(_partition_path(results, 1), "results") + status = deserialize_status(job_results_path) + if status != RunStatus.SUCCESSFUL + @warn "partition job index = $i was not successful: $status" + overall_status = status + end + end + + return overall_status +end + +function _merge_store_files!(results::SimulationPartitionResults) + HDF5.h5open(_store_path(results), "r+") do dst + for i in 1:get_num_partitions(results.partitions) + HDF5.h5open(joinpath(_partition_path(results, i), _store_subpath()), "r") do src + _copy_datasets!(results, i, src, dst) + end + end + + for dataset in values(results.datasets) + if occursin("decision_models", HDF5.name(dataset)) + IS.@assert_op HDF5.attrs(dataset)[_TEMP_WRITE_POSITION] == + size(dataset)[end] + 1 + else + IS.@assert_op HDF5.attrs(dataset)[_TEMP_WRITE_POSITION] == + size(dataset)[1] + 1 + end + delete!(HDF5.attrs(dataset), _TEMP_WRITE_POSITION) + end + end +end + +function _copy_datasets!( + results::SimulationPartitionResults, + index::Int, + src::HDF5.File, + dst::HDF5.File, +) + output_types = string.(STORE_CONTAINERS) + + function process_dataset(src_dataset, merge_func) + if !endswith(HDF5.name(src_dataset), "__columns") + name = HDF5.name(src_dataset) + dst_dataset = dst[name] + if !haskey(results.datasets, name) + results.datasets[name] = dst_dataset + HDF5.attrs(dst_dataset)[_TEMP_WRITE_POSITION] = 1 + end + merge_func(results, index, src_dataset, dst_dataset) + end + end + + for src_group in src["simulation/decision_models"] + for output_type in output_types + for src_dataset in src_group[output_type] + process_dataset(src_dataset, _merge_dataset_rows!) + end + end + process_dataset(src_group["optimizer_stats"], _merge_dataset_rows!) + end + + for output_type in output_types + for src_dataset in src["simulation/emulation_model/$output_type"] + process_dataset(src_dataset, _merge_dataset_columns!) + end + end +end + +function _merge_dataset_columns!(results::SimulationPartitionResults, index, src, dst) + num_columns = size(src)[1] + step_range = get_absolute_step_range(results.partitions, index) + IS.@assert_op num_columns % length(step_range) == 0 + num_columns_per_step = num_columns ÷ length(step_range) + skip_offset = get_valid_step_offset(results.partitions, index) - 1 + src_start = 1 + num_columns_per_step * skip_offset + len = get_valid_step_length(results.partitions, index) * num_columns_per_step + src_end = src_start + len - 1 + + IS.@assert_op ndims(src) == ndims(dst) + dst_start = HDF5.attrs(dst)[_TEMP_WRITE_POSITION] + if ndims(src) == 2 + IS.@assert_op size(src)[2] == size(dst)[2] + dst_end = dst_start + len - 1 + dst[dst_start:dst_end, :] = src[src_start:src_end, :] + else + error("Unsupported dataset ndims: $(ndims(src))") + end + + HDF5.attrs(dst)[_TEMP_WRITE_POSITION] = dst_end + 1 + return +end + +function _merge_dataset_rows!(results::SimulationPartitionResults, index, src, dst) + num_rows = size(src)[end] + step_range = get_absolute_step_range(results.partitions, index) + IS.@assert_op num_rows % length(step_range) == 0 + num_rows_per_step = num_rows ÷ length(step_range) + skip_offset = get_valid_step_offset(results.partitions, index) - 1 + src_start = 1 + num_rows_per_step * skip_offset + len = get_valid_step_length(results.partitions, index) * num_rows_per_step + src_end = src_start + len - 1 + + IS.@assert_op ndims(src) == ndims(dst) + dst_start = HDF5.attrs(dst)[_TEMP_WRITE_POSITION] + if ndims(src) == 2 + IS.@assert_op size(src)[1] == size(dst)[1] + dst_end = dst_start + len - 1 + dst[:, dst_start:dst_end] = src[:, src_start:src_end] + elseif ndims(src) == 3 + IS.@assert_op size(src)[1] == size(dst)[1] + IS.@assert_op size(src)[2] == size(dst)[2] + dst_end = dst_start + len - 1 + IS.@assert_op dst_end <= size(dst)[3] + dst[:, :, dst_start:dst_end] = src[:, :, src_start:src_end] + else + error("Unsupported dataset ndims: $(ndims(src))") + end + + HDF5.attrs(dst)[_TEMP_WRITE_POSITION] = dst_end + 1 + return +end + +function _complete(results::SimulationPartitionResults, status) + serialize_status(status, joinpath(results.path, "results")) + store_path = _store_path(results) + compute_file_hash(dirname(store_path), basename(store_path)) + return +end diff --git a/src/simulation/simulation_partitions.jl b/src/simulation/simulation_partitions.jl new file mode 100644 index 0000000000..b65c06f0fc --- /dev/null +++ b/src/simulation/simulation_partitions.jl @@ -0,0 +1,282 @@ +""" +Defines how a simulation can be partition into partitions and run in parallel. +""" +struct SimulationPartitions <: IS.InfrastructureSystemsType + "Number of steps in the simulation" + num_steps::Int + "Number of steps in each partition" + period::Int + "Number of steps that a partition overlaps with the previous partition" + num_overlap_steps::Int + + function SimulationPartitions(num_steps, period, num_overlap_steps=1) + if num_overlap_steps > period + error( + "period=$period must be greater than num_overlap_steps=$num_overlap_steps", + ) + end + if period >= num_steps + error("period=$period must be less than simulation steps=$num_steps") + end + return new(num_steps, period, num_overlap_steps) + end +end + +function SimulationPartitions(; num_steps, period, num_overlap_steps) + return SimulationPartitions(num_steps, period, num_overlap_steps) +end + +""" +Return the number of partitions in the simulation. +""" +get_num_partitions(x::SimulationPartitions) = Int(ceil(x.num_steps / x.period)) + +""" +Return a UnitRange for the steps in the partition with the given index. Includes overlap. +""" +function get_absolute_step_range(partitions::SimulationPartitions, index::Int) + num_partitions = _check_partition_index(partitions, index) + start_index = partitions.period * (index - 1) + 1 + if index < num_partitions + end_index = start_index + partitions.period - 1 + else + end_index = partitions.num_steps + end + + if index > 1 + start_index -= partitions.num_overlap_steps + end + + return start_index:end_index +end + +""" +Return the step offset for valid data at the given index. +""" +function get_valid_step_offset(partitions::SimulationPartitions, index::Int) + _check_partition_index(partitions, index) + return index == 1 ? 1 : partitions.num_overlap_steps + 1 +end + +""" +Return the length of valid data at the given index. +""" +function get_valid_step_length(partitions::SimulationPartitions, index::Int) + num_partitions = _check_partition_index(partitions, index) + if index < num_partitions + return partitions.period + end + + remainder = partitions.num_steps % partitions.period + return remainder == 0 ? partitions.period : remainder +end + +function _check_partition_index(partitions::SimulationPartitions, index::Int) + num_partitions = get_num_partitions(partitions) + if index <= 0 || index > num_partitions + error("index=$index=inde must be > 0 and <= $num_partitions") + end + + return num_partitions +end + +function IS.serialize(partitions::SimulationPartitions) + return IS.serialize_struct(partitions) +end + +function process_simulation_partition_cli_args(build_function, execute_function, args...) + length(args) < 2 && error("Usage: setup|execute|join [options]") + function config_logging(filename) + return IS.configure_logging( + console=true, + console_stream=stderr, + console_level=Logging.Warn, + file=true, + filename=filename, + file_level=Logging.Info, + file_mode="w", + tracker=nothing, + set_global=true, + ) + end + + function throw_if_missing(actual, required, label) + diff = setdiff(required, actual) + !isempty(diff) && error("Missing required options for $label: $diff") + end + + operation = args[1] + options = Dict{String, String}() + for opt in args[2:end] + !startswith(opt, "--") && error("All options must start with '--': $opt") + fields = split(opt[3:end], "=") + length(fields) != 2 && error("All options must use the format --name=value: $opt") + options[fields[1]] = fields[2] + end + + if haskey(options, "output-dir") + output_dir = options["output-dir"] + elseif haskey(ENV, "JADE_RUNTIME_OUTPUT") + output_dir = joinpath(ENV["JADE_RUNTIME_OUTPUT"], "job-outputs") + else + error("output-dir must be specified as a CLI option or environment variable") + end + + if operation == "setup" + required = Set(("simulation-name", "num-steps", "num-period-steps")) + throw_if_missing(keys(options), required, operation) + if !haskey(options, "num-overlap-steps") + options["num-overlap-steps"] = "0" + end + + num_steps = parse(Int, options["num-steps"]) + num_period_steps = parse(Int, options["num-period-steps"]) + num_overlap_steps = parse(Int, options["num-overlap-steps"]) + partitions = SimulationPartitions(num_steps, num_period_steps, num_overlap_steps) + config_logging(joinpath(output_dir, "setup_partition_simulation.log")) + build_function(output_dir, options["simulation-name"], partitions) + elseif operation == "execute" + throw_if_missing(keys(options), Set(("simulation-name", "index")), operation) + index = parse(Int, options["index"]) + base_dir = joinpath(output_dir, options["simulation-name"]) + partition_output_dir = joinpath(base_dir, "simulation_partitions", string(index)) + config_file = joinpath(base_dir, "simulation_partitions", "config.json") + config = open(config_file, "r") do io + JSON3.read(io, Dict) + end + partitions = IS.deserialize(SimulationPartitions, config) + config_logging(joinpath(partition_output_dir, "run_partition_simulation.log")) + sim = build_function( + partition_output_dir, + options["simulation-name"], + partitions, + index, + ) + execute_function(sim) + elseif operation == "join" + throw_if_missing(keys(options), Set(("simulation-name",)), operation) + base_dir = joinpath(output_dir, options["simulation-name"]) + config_file = joinpath(base_dir, "simulation_partitions", "config.json") + config = open(config_file, "r") do io + JSON3.read(io, Dict) + end + partitions = IS.deserialize(SimulationPartitions, config) + config_logging(joinpath(base_dir, "logs", "join_partitioned_simulation.log")) + join_simulation(base_dir) + else + error("Unsupported operation=$operation") + end + + return +end + +""" +Run a partitioned simulation in parallel on a local computer. + +# Arguments + + - `build_function`: Function reference that returns a built Simulation. + - `execute_function`: Function reference that executes a Simulation. + - `script::AbstractString`: Path to script that includes ``build_function`` and ``execute_function``. + - `output_dir::AbstractString`: Path for simulation outputs + - `name::AbstractString`: Simulation name + - `num_steps::Integer`: Total number of steps in the simulation + - `period::Integer`: Number of steps in each simulation partition + - `num_overlap_steps::Integer`: Number of steps that each partition overlaps with the previous partition + - `num_parallel_processes`: Number of partitions to run in parallel. If nothing, use the number of cores. + - `exeflags`: Path to Julia project. Forwarded to Distributed.addprocs. + - `force`: Overwrite the output directory if it already exists. +""" +function run_parallel_simulation( + build_function, + execute_function; + script::AbstractString, + output_dir::AbstractString, + name::AbstractString, + num_steps::Integer, + period::Integer, + num_overlap_steps::Integer=1, + num_parallel_processes=nothing, + exeflags=nothing, + force=false, +) + if isnothing(num_parallel_processes) + num_parallel_processes = Sys.CPU_THREADS + end + + partitions = SimulationPartitions(num_steps, period, num_overlap_steps) + num_partitions = get_num_partitions(partitions) + if isdir(output_dir) + if !force + error( + "output_dir=$output_dir already exists. Choose a different name or set force=true.", + ) + end + rm(output_dir, recursive=true) + end + mkdir(output_dir) + @info "Run parallel simulation" name script output_dir num_steps num_partitions num_parallel_processes + + args = [ + "setup", + "--simulation-name=$name", + "--num-steps=$(partitions.num_steps)", + "--num-period-steps=$(partitions.period)", + "--num-overlap-steps=$(partitions.num_overlap_steps)", + "--output-dir=$output_dir", + ] + parent_module_name = nameof(parentmodule(build_function)) + build_func_name = nameof(build_function) + execute_func_name = nameof(execute_function) + process_simulation_partition_cli_args(build_function, execute_function, args...) + jobs = Vector{Dict}(undef, num_partitions) + for i in 1:num_partitions + args = Dict( + "parent_module" => parent_module_name, + "build_function" => build_func_name, + "execute_function" => execute_func_name, + "args" => [ + "execute", + "--simulation-name=$name", + "--index=$i", + "--output-dir=$output_dir", + ], + ) + jobs[i] = args + end + + if isnothing(exeflags) + Distributed.addprocs(num_parallel_processes) + else + Distributed.addprocs(num_parallel_processes, exeflags=exeflags) + end + + Distributed.@everywhere include($script) + try + Distributed.pmap(PowerSimulations._run_parallel_simulation, jobs) + finally + Distributed.rmprocs(Distributed.workers()...) + end + + args = ["join", "--simulation-name=$name", "--output-dir=$output_dir"] + process_simulation_partition_cli_args(build_function, execute_function, args...) +end + +function _run_parallel_simulation(params) + start = time() + if params["parent_module"] == :Main + parent_module = Main + else + # TODO: not tested + parent_module = Base.root_module(Base.__toplevel__, Symbol(params["parent_module"])) + end + result = process_simulation_partition_cli_args( + getproperty(parent_module, params["build_function"]), + getproperty(parent_module, params["execute_function"]), + params["args"]..., + ) + duration = time() - start + args = params["args"] + @info "Completed partition" args duration + return result +end diff --git a/src/simulation/simulation_results.jl b/src/simulation/simulation_results.jl index 7aabd40b28..60451ae476 100644 --- a/src/simulation/simulation_results.jl +++ b/src/simulation/simulation_results.jl @@ -1,8 +1,10 @@ function check_folder_integrity(folder::String) folder_files = readdir(folder) - alien_files = filter(!∈(KNOWN_SIMULATION_PATHS), folder_files) + alien_files = setdiff(folder_files, KNOWN_SIMULATION_PATHS) if isempty(alien_files) return true + else + @warn "Unrecognized simulation files: $(sort(alien_files))" end if "data_store" ∉ folder_files error("The file path doesn't contain any data_store folder") @@ -83,7 +85,6 @@ function SimulationResults(path::AbstractString, execution=nothing; ignore_statu decision_problem_results[name] = problem_result end - emulation_params = get_emulation_model_params(sim_params) emulation_result = SimulationProblemResults( EmulationModel, store, diff --git a/test/run_partitioned_simulation.jl b/test/run_partitioned_simulation.jl new file mode 100644 index 0000000000..8d71dfda68 --- /dev/null +++ b/test/run_partitioned_simulation.jl @@ -0,0 +1,171 @@ +using PowerSimulations +using PowerSystems +using PowerSystemCaseBuilder +using InfrastructureSystems +import PowerSystemCaseBuilder: PSITestSystems +using Logging +using Test + +using PowerModels +using DataFrames +using Dates +using JuMP +using TimeSeries +using ParameterJuMP +using CSV +using DataFrames +using DataStructures +import UUIDs +using Random +import Serialization + +const PM = PowerModels +const PSY = PowerSystems +const PSI = PowerSimulations +const PSB = PowerSystemCaseBuilder + +const PJ = ParameterJuMP +const IS = InfrastructureSystems +const BASE_DIR = string(dirname(dirname(pathof(PowerSimulations)))) +const DATA_DIR = joinpath(BASE_DIR, "test/test_data") + +include(joinpath(BASE_DIR, "test/test_utils/common_operation_model.jl")) +include(joinpath(BASE_DIR, "test/test_utils/model_checks.jl")) +include(joinpath(BASE_DIR, "test/test_utils/mock_operation_models.jl")) +include(joinpath(BASE_DIR, "test/test_utils/solver_definitions.jl")) +include(joinpath(BASE_DIR, "test/test_utils/operations_problem_templates.jl")) + +function build_simulation( + output_dir::AbstractString, + simulation_name::AbstractString, + partitions::Union{Nothing, SimulationPartitions}=nothing, + index::Union{Nothing, Integer}=nothing; + initial_time=nothing, + num_steps=nothing, +) + if isnothing(partitions) && isnothing(num_steps) + error("num_steps must be set if partitions is nothing") + end + if !isnothing(partitions) && !isnothing(num_steps) + error("num_steps and partitions cannot both be set") + end + c_sys5_pjm_da = PSB.build_system(PSITestSystems, "c_sys5_pjm") + PSY.transform_single_time_series!(c_sys5_pjm_da, 48, Hour(24)) + c_sys5_pjm_rt = PSB.build_system(PSITestSystems, "c_sys5_pjm_rt") + PSY.transform_single_time_series!(c_sys5_pjm_rt, 12, Hour(1)) + + for sys in [c_sys5_pjm_da, c_sys5_pjm_rt] + th = get_component(ThermalStandard, sys, "Park City") + set_active_power_limits!(th, (min=0.1, max=1.7)) + set_status!(th, false) + set_active_power!(th, 0.0) + c = get_operation_cost(th) + c.start_up = 1500 + c.shut_down = 75 + set_time_at_status!(th, 1) + + th = get_component(ThermalStandard, sys, "Alta") + set_time_limits!(th, (up=5, down=1)) + set_active_power_limits!(th, (min=0.05, max=0.4)) + set_active_power!(th, 0.05) + c = get_operation_cost(th) + c.start_up = 400 + c.shut_down = 200 + set_time_at_status!(th, 2) + + th = get_component(ThermalStandard, sys, "Brighton") + set_active_power_limits!(th, (min=2.0, max=6.0)) + c = get_operation_cost(th) + set_active_power!(th, 4.88041) + c.start_up = 5000 + c.shut_down = 3000 + + th = get_component(ThermalStandard, sys, "Sundance") + set_active_power_limits!(th, (min=1.0, max=2.0)) + set_time_limits!(th, (up=5, down=1)) + set_active_power!(th, 2.0) + c = get_operation_cost(th) + c.start_up = 4000 + c.shut_down = 2000 + set_time_at_status!(th, 1) + + th = get_component(ThermalStandard, sys, "Solitude") + set_active_power_limits!(th, (min=1.0, max=5.2)) + set_ramp_limits!(th, (up=0.0052, down=0.0052)) + set_active_power!(th, 2.0) + c = get_operation_cost(th) + c.start_up = 3000 + c.shut_down = 1500 + end + + to_json(c_sys5_pjm_da, "PSI-5-BUS-UC-ED/c_sys5_pjm_da.json"; force=true) + to_json(c_sys5_pjm_rt, "PSI-5-BUS-UC-ED/c_sys5_pjm_rt.json"; force=true) + + HiGHSoptimizer = optimizer_with_attributes(HiGHS.Optimizer) + + template_uc = template_unit_commitment() + set_network_model!( + template_uc, + NetworkModel( + StandardPTDFModel; + PTDF=PTDF(c_sys5_pjm_da), + # duals = [CopperPlateBalanceConstraint] + ), + ) + + set_device_model!(template_uc, ThermalStandard, ThermalStandardUnitCommitment) + template_ed = deepcopy(template_uc) + set_device_model!(template_ed, ThermalStandard, ThermalStandardDispatch) + + models = SimulationModels( + decision_models=[ + DecisionModel( + template_uc, + c_sys5_pjm_da, + optimizer=HiGHSoptimizer, + name="UC", + initialize_model=false, + ), + DecisionModel( + template_ed, + c_sys5_pjm_rt, + optimizer=HiGHSoptimizer, + name="ED", + calculate_conflict=false, + ), + ], + ) + sequence = SimulationSequence( + models=models, + feedforwards=Dict( + "ED" => [ + SemiContinuousFeedforward( + component_type=ThermalStandard, + source=OnVariable, + affected_values=[ActivePowerVariable], + ), + ], + ), + ini_cond_chronology=InterProblemChronology(), + ) + + sim = Simulation( + name=simulation_name, + steps=isnothing(partitions) ? num_steps : partitions.num_steps, + models=models, + sequence=sequence, + simulation_folder=output_dir, + initial_time=initial_time, + ) + + status = build!(sim; partitions=partitions, index=index, serialize=isnothing(index)) + if status != PSI.BuildStatus.BUILT + error("Failed to build simulation: status=$status") + end + + return sim +end + +function execute_simulation(sim, args...; kwargs...) + return execute!(sim) +end diff --git a/test/test_simulation_partitions.jl b/test/test_simulation_partitions.jl new file mode 100644 index 0000000000..9344f0e7ad --- /dev/null +++ b/test/test_simulation_partitions.jl @@ -0,0 +1,109 @@ +@testset "Test partitions and step ranges" begin + partitions = SimulationPartitions(2, 1, 0) + @test PSI.get_absolute_step_range(partitions, 1) == 1:1 + @test PSI.get_valid_step_offset(partitions, 1) == 1 + @test PSI.get_valid_step_length(partitions, 1) == 1 + @test PSI.get_absolute_step_range(partitions, 2) == 2:2 + @test PSI.get_valid_step_offset(partitions, 2) == 1 + @test PSI.get_valid_step_length(partitions, 2) == 1 + + partitions = SimulationPartitions(365, 7, 1) + @test get_num_partitions(partitions) == 53 + @test PSI.get_absolute_step_range(partitions, 1) == 1:7 + @test PSI.get_valid_step_offset(partitions, 1) == 1 + @test PSI.get_valid_step_length(partitions, 1) == 7 + @test PSI.get_absolute_step_range(partitions, 2) == 7:14 + @test PSI.get_valid_step_offset(partitions, 2) == 2 + @test PSI.get_valid_step_length(partitions, 2) == 7 + @test PSI.get_absolute_step_range(partitions, 52) == 357:364 + @test PSI.get_valid_step_offset(partitions, 52) == 2 + @test PSI.get_valid_step_length(partitions, 52) == 7 + @test PSI.get_absolute_step_range(partitions, 53) == 364:365 + @test PSI.get_valid_step_offset(partitions, 53) == 2 + @test PSI.get_valid_step_length(partitions, 53) == 1 + + @test_throws ErrorException PSI.get_absolute_step_range(partitions, -1) + @test_throws ErrorException PSI.get_absolute_step_range(partitions, 54) +end + +@testset "Test simulation partitions" begin + sim_dir = mktempdir() + script = joinpath(BASE_DIR, "test", "run_partitioned_simulation.jl") + include(script) + + partition_name = "partitioned" + run_parallel_simulation( + build_simulation, + execute_simulation, + script=script, + output_dir=sim_dir, + name=partition_name, + num_steps=3, + period=1, + num_overlap_steps=1, + num_parallel_processes=3, + exeflags="--project=test", + force=true, + ) + + regular_name = "regular" + regular_sim = build_simulation( + sim_dir, + regular_name, + initial_time=DateTime("2024-01-02T00:00:00"), + num_steps=1, + ) + @test execute_simulation(regular_sim) == PSI.RunStatus.SUCCESSFUL + + regular_results = SimulationResults(joinpath(sim_dir, regular_name)) + partitioned_results = SimulationResults(joinpath(sim_dir, partition_name)) + + functions = ( + read_realized_aux_variables, + read_realized_duals, + read_realized_expressions, + read_realized_parameters, + read_realized_variables, + ) + key_strings_to_skip = ("Flow", "On", "Off", "Start", "Stop") + for name in ("ED", "UC") + regular_model_results = get_decision_problem_results(regular_results, name) + partitioned_model_results = get_decision_problem_results(partitioned_results, name) + + for func in functions + regular = func(regular_model_results) + partitioned = func(partitioned_model_results) + @test sort(collect(keys(regular))) == sort(collect(keys(partitioned))) + for key in keys(regular) + t_start = regular[key][1, 1] + t_end = regular[key][end, 1] + rdf = regular[key] + pdf = partitioned[key] + pdf = pdf[(pdf.DateTime .>= t_start) .& (pdf.DateTime .<= t_end), :] + @test nrow(rdf) == nrow(pdf) + @test ncol(rdf) == ncol(pdf) + skip = false + for key_string_to_skip in key_strings_to_skip + if occursin(key_string_to_skip, key) + skip = true + break + end + end + skip && continue + r_sum = 0 + p_sum = 0 + atol = occursin("ProductionCostExpression", key) ? 11000 : 0 + for i in 2:ncol(rdf) + r_sum += sum(rdf[!, i]) + p_sum += sum(pdf[!, i]) + end + if !isapprox(r_sum, p_sum, atol=atol) + @error "Mismatch" r_sum p_sum key + end + @test isapprox(r_sum, p_sum, atol=atol) + end + end + end + + # TODO: Can emulation model results be validated? +end From 605992c8282f634dcc8d57f67e1188e1132615fb Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 10 Aug 2022 08:54:15 -0600 Subject: [PATCH 2/4] Reduce number of parallel processes GitHub CI was likely running out of memory or CPU. --- test/test_simulation_partitions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_simulation_partitions.jl b/test/test_simulation_partitions.jl index 9344f0e7ad..b8688146d1 100644 --- a/test/test_simulation_partitions.jl +++ b/test/test_simulation_partitions.jl @@ -41,7 +41,7 @@ end num_steps=3, period=1, num_overlap_steps=1, - num_parallel_processes=3, + num_parallel_processes=1, exeflags="--project=test", force=true, ) From 2024ea3131843e0514c84f91287a5e921645d543 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 10 Aug 2022 10:39:57 -0600 Subject: [PATCH 3/4] Make num_parallel_processes dependent on CI env --- test/test_simulation_partitions.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_simulation_partitions.jl b/test/test_simulation_partitions.jl index b8688146d1..827bb9db8c 100644 --- a/test/test_simulation_partitions.jl +++ b/test/test_simulation_partitions.jl @@ -41,7 +41,8 @@ end num_steps=3, period=1, num_overlap_steps=1, - num_parallel_processes=1, + # Running multiple processes in CI can kill the VM. + num_parallel_processes=haskey(ENV, "CI") ? 1 : 3, exeflags="--project=test", force=true, ) From 3ff25647fe951137a0785535eef23e8c283eeb32 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 10 Aug 2022 15:38:05 -0600 Subject: [PATCH 4/4] Add test --- test/test_utils.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_utils.jl b/test/test_utils.jl index 7940c0723f..3709dbf639 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -39,3 +39,12 @@ end df = PSI.axis_array_to_dataframe(sparse_valid, mock_key) @test size(df) == (24, 55) end + +@testset "Test simulation output directory name" begin + tmpdir = mktempdir() + name = "simulation" + dir1 = mkdir(joinpath(tmpdir, name)) + @test PSI._get_output_dir_name(tmpdir, name) == joinpath(tmpdir, name * "-2") + dir2 = mkdir(joinpath(tmpdir, name * "-2")) + @test PSI._get_output_dir_name(tmpdir, name) == joinpath(tmpdir, name * "-3") +end