diff --git a/docs/make.jl b/docs/make.jl index 6b72df4c..5646a651 100755 --- a/docs/make.jl +++ b/docs/make.jl @@ -28,7 +28,7 @@ makedocs( "FilterExps" => "submodules/experiments/FilterExps.md", "SmootherExps" => "submodules/experiments/SmootherExps.md", "SingleExperimentDriver" => "submodules/experiments/SingleExperimentDriver.md", - "Slurm" => "submodules/experiments/Slurm.md", + "ParallelExperimentDriver" => "submodules/experiments/ParallelExperimentDriver.md", ], "Analysis" => Any[ "ProcessExperimentData" => "submodules/analysis/ProcessExperimentData.md", @@ -38,7 +38,6 @@ makedocs( ] ) - deploydocs( repo = "github.com:cgrudz/DataAssimilationBenchmarks.jl.git", ) diff --git a/docs/src/index.md b/docs/src/index.md index 3fadf621..2fcad2c4 100755 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -5,7 +5,7 @@ validating sequential filters and smoothers in toy model twin experiments. This code is meant to be performant in the sense that large hyper-parameter discretizations can be explored to determine hyper-parameter sensitivity and reliability of results across different experimental regimes, with parallel implementations in native Julia distributed -computing and using workload managers such as Slurm. +computing. This package currently includes code for developing and testing data assimilation schemes in the [L96-s model](https://gmd.copernicus.org/articles/13/1903/2020/) and the IEEE 39 bus test diff --git a/docs/src/submodules/experiments/ParallelExperimentDriver.md b/docs/src/submodules/experiments/ParallelExperimentDriver.md index 3c51c6c0..5ce5fa88 100755 --- a/docs/src/submodules/experiments/ParallelExperimentDriver.md +++ b/docs/src/submodules/experiments/ParallelExperimentDriver.md @@ -2,12 +2,14 @@ ## ParallelExperimentDriver API -The `ParallelExperimentDriver.jl` is a simple parallel implementation, though currently lacks a soft-fail when numerical -instability is encountered. This means that if a single experiment configuration in the collection fails due to overflow, the entire -collection will cancel. A fix for this is being explored, but the recommendation is to use the slurm submit -scripts below as templates for generating large parameter grid configurations and running them on servers. +The `ParallelExperimentDriver.jl` is a simple parallel implementation of calling experiment parameter arrays +with `pmap` and Julia's native distributed computing. This defines argumentless functions to construct +the parameter array and input data necessary to generate a sensitivity test, and implements a soft-fail for +experiments instability is encountered causing a crash of an experiment. This means that if a single experiment +configuration in the parameter array fails due to overflow, the remaining configurations will continue their +own course unaffected. -## Docstrings -#```@autodocs -#Modules = [DataAssimilationBenchmarks.ParallelExperimentDriver] -#``` +## Methods +```@autodocs +Modules = [DataAssimilationBenchmarks.ParallelExperimentDriver] +``` diff --git a/docs/src/submodules/experiments/SmootherExps.md b/docs/src/submodules/experiments/SmootherExps.md index ada517d0..0464f7f4 100755 --- a/docs/src/submodules/experiments/SmootherExps.md +++ b/docs/src/submodules/experiments/SmootherExps.md @@ -1,38 +1,38 @@ # SmootherExps - ## SmootherExps API - -The `SmootherExps.jl` sub-module configures twin experiments using a stored time series as generated by -[GenerateTimeSeries](@ref) to generate hyper-parameter configurations for the estimtor with the same -underlying observation generating process. -Experiment configurations are generated by function calls as with the filter experiments, but with the additional -options of how the outer-loop is configured with a classic, single-iteration or the fully iterative Gauss-Newton style smoother. -The parameters `lag` and `shift` specify how the data assimilation windows are translated in over the observation -and analysis times. The `mda` parameter is only applicable to the single-iteration and Gauss-Newton style smoothers, -utlizing sequential multiple data assimilation. Note, the single-iteration and fully iterative Gauss-Newton style smoothers are -only defined for MDA compatible values of lag and shift where the lag is an integer multiple of the shift. - -Currently debugged and validated smoother experiment configurations include +The `SmootherExps.jl` sub-module contains methods to configure filter twin experiments, +using a stored time series as generated by [GenerateTimeSeries](@ref) as the underlying +observation generating process. The frequency of observations in continuous time is defined +by the frequency of data saved in the time series and is inferred by the experiment +when reading in the data. + +```@raw html +
+Observation analysis forecast cycle over multiple data assimilation windows +
``` -classic_state -- classic EnKS style state estimation -classic_param -- classic EnKS style state-parameter estimation -single_iteration_state -- single-iteration EnKS state estimation -single_iteration_param -- single-iteration EnKS state-parameter estimation -iterative_state -- Gauss-Newton style state estimation -iterative_param -- Gauss-Newton style state-parameter estimation -``` -Other techniques are still in debugging and validation. Each of these takes an analysis type as used in the -`transform` function in the `EnsembleKalmanSchemes.jl` sub-module, like the filter analyses in the filter experiments. - +Smoother experiment configurations are generated by supplying a +[NamedTuple](https://docs.julialang.org/en/v1/base/base/#Core.NamedTuple) +with the required fields as specified in the experiment method. Conventions for +these arguments are the same as with the [FilterExps](@ref), with the additional options +that configure the data assimilation window (DAW) and how this is shifted in time: + * `lag` - the number of past observation / analysis times to reanalyze in a DAW, corresponding to ``L`` in the figure above; + * `shift`- the nunber of observation / analysis times to shift the DAW, corresponding to ``S`` in the figure above; + * `mda` - (__Multiple Data Assimilation__), type `Bool`, determines whether the technique of multiple data assimilation is used (only compatible with `single_iteration` and `iterative` smoothers. -All experiments are funcitonalized so that they can be called from an array of parameter -values which will typically be varied with naive parallelism. Relevant arguments and -experimental results are dumped as a side effect to a dictionary in a JLD2. -Experiments return their runtime in minutes. +Currently debugged and validated smoother experiment configurations include + * `classic_state` - classic ETKS style state estimation + * `classic_param` - classic ETKS style state-parameter estimation + * `single_iteration_state` - SIEnKS state estimation + * `single_iteration_param` - SIEnKS state-parameter estimation + * `iterative_state` - IEnKS Gauss-Newton style state estimation + * `iterative_param` - IEnKS Gauss-Newton style state-parameter estimation +Note, the single-iteration and fully iterative Gauss-Newton style smoothers are only defined +for MDA compatible values of lag and shift where the lag is an integer multiple of the shift. -## Docstrings +## Smoother Experiment Methods ```@autodocs Modules = [DataAssimilationBenchmarks.SmootherExps] ``` diff --git a/docs/src/submodules/methods/EnsembleKalmanSchemes.md b/docs/src/submodules/methods/EnsembleKalmanSchemes.md index 3ff31fc4..121e9663 100755 --- a/docs/src/submodules/methods/EnsembleKalmanSchemes.md +++ b/docs/src/submodules/methods/EnsembleKalmanSchemes.md @@ -4,12 +4,8 @@ There are currently four families of ensemble Kalman estimators available in this package, which define the outer-loop of the data assimilation cycle. Particularly, these define how the sequential data assimilation cycle will pass over a time series of observations, -conceptually pictured in the figure below. -```@raw html -
-Observation analysis forecast cycle over multiple data assimilation windows -
-``` +with more details in the [SmootherExps](@ref) documents. + Ensemble filters only produce analyses forward-in-time. The classic lag-shift smoother runsi identically to the filter in its forecast and filter steps, but includes an additional retrospective analysis to past ensemble states stored in memory. The single iteration smoother follows diff --git a/src/DataAssimilationBenchmarks.jl b/src/DataAssimilationBenchmarks.jl index e4653e23..cc58f38d 100755 --- a/src/DataAssimilationBenchmarks.jl +++ b/src/DataAssimilationBenchmarks.jl @@ -151,6 +151,7 @@ using .D3VARExps export DeSolvers, EnsembleKalmanSchemes, XdVAR, L96, IEEE39bus, ObsOperators, GenerateTimeSeries, FilterExps, SingleExperimentDriver, ParallelExperimentDriver, D3VARExps + ############################################################################################## # info diff --git a/src/experiments/FilterExps.jl b/src/experiments/FilterExps.jl index e4348f2a..e476cca2 100755 --- a/src/experiments/FilterExps.jl +++ b/src/experiments/FilterExps.jl @@ -40,6 +40,10 @@ Output from the experiment is saved in a dictionary of the form, "s_infl" => round(s_infl, digits=2) ) + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"]::Array{Float64,2} + end + Experiment output is written to a directory defined by path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "/" @@ -256,6 +260,11 @@ Output from the experiment is saved in a dictionary of the form, "s_infl" => round(s_infl, digits=2), "p_infl" => round(p_infl, digits=2) ) + + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"]::Array{Float64,2} + end + Experiment output is written to a directory defined by path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "/" @@ -465,7 +474,6 @@ function ensemble_filter_param((time_series, method, seed, nanl, obs_un, obs_dim "p_infl" => round(p_infl, digits=2) ) - # check if there is a diffusion structure matrix if haskey(ts, "diff_mat") data["diff_mat"] = ts["diff_mat"]::Array{Float64,2} end diff --git a/src/experiments/ParallelExperimentDriver.jl b/src/experiments/ParallelExperimentDriver.jl index 3bcf710f..a2005e71 100755 --- a/src/experiments/ParallelExperimentDriver.jl +++ b/src/experiments/ParallelExperimentDriver.jl @@ -2,140 +2,300 @@ module ParallelExperimentDriver ############################################################################################## # imports and exports -using Distributed -@everywhere push!(LOAD_PATH, "/DataAssimilationBenchmarks/src/") -@everywhere push!(LOAD_PATH, "/DataAssimilationBenchmarks/src/methods") -@everywhere push!(LOAD_PATH, "/DataAssimilationBenchmarks/src/models") -@everywhere push!(LOAD_PATH, "/DataAssimilationBenchmarks/src/experiments") -@everywhere using JLD2, ..DataAssimilationBenchmarks, ..FilterExps, ..SmootherExps, - ..EnsembleKalmanSchemes, ..DeSolvers, ..L96, ..ParallelExperimentDriver -@everywhere export experiment -#@everywhere export wrap_exp +using ..DataAssimilationBenchmarks +export ensemble_filter_adaptive_inflation, ensemble_filter_param, classic_ensemble_state, + classic_ensemble_param, single_iteration_ensemble_state, iterative_ensemble_state ############################################################################################## -## Timeseries data +# Utility methods and definitions ############################################################################################## -# observation timeseries to load into the experiment as truth twin, timeseries are named by -# the model, seed to initialize, the integration scheme used to produce, number of analyses, -# the spinup length, and the time length between observation points - -ts1 = "../data/time_series/L96_time_series_seed_0000_dim_40_diff_0.00_F_08.0_tanl_0.05" * - "_nanl_50000_spin_5000_h_0.050.jld2" -ts2 = "../data/time_series/L96_time_series_seed_0000_dim_40_diff_0.00_F_08.0_tanl_0.10" * - "_nanl_50000_spin_5000_h_0.050.jld2" -ts3 = "../data/time_series/L96_time_series_seed_0000_dim_40_diff_0.00_F_08.0_tanl_0.15" * - "_nanl_50000_spin_5000_h_0.050.jld2" -ts4 = "../data/time_series/L96_time_series_seed_0000_dim_40_diff_0.00_F_08.0_tanl_0.20" * - "_nanl_50000_spin_5000_h_0.050.jld2" -ts5 = "../data/time_series/L96_time_series_seed_0000_dim_40_diff_0.00_F_08.0)tanl_0.25" * - "_nanl_50000_spin_5000_h_0.050.jld2" -############################################################################################## -## Experiment parameter generation -############################################################################################## +path = pkgdir(DataAssimilationBenchmarks) * "/src/data/time_series/" ############################################################################################## # Filters ############################################################################################## -## filter_state -# -## [time_series, scheme, seed, nanl, obs_un, obs_dim, N_ens, infl] = args -# -#schemes = ["enkf-n-primal", "enkf-n-primal-ls", "enkf-n-dual"] -#seed = 0 -#obs_un = 1.0 -#obs_dim = 40 -#N_ens = 15:43 -#infl = [1.0]#LinRange(1.0, 1.20, 21) -#nanl = 2500 -# -## load the experiments -#args = Tuple[] -#for scheme in schemes -# for N in N_ens -# for α in infl -# tmp = (time_series, scheme, seed, nanl, obs_un, obs_dim, N, α) -# push!(args, tmp) -# end -# end -#end -# -#experiment = FilterExps.filter_state -# -# +# compare adaptive inflation methods + +""" + args, exp = ensemble_filter_adaptive_inflation() + +Constucts a parameter map and experiment wrapper for sensitivity test of adaptive inflation. +""" +function ensemble_filter_adaptive_inflation() + + exp = DataAssimilationBenchmarks.FilterExps.ensemble_filter_state + function wrap_exp(arguments) + try + exp(arguments) + catch + print("Error on " * string(arguments) * "\n") + end + end + + # set time series parameters + seed = 123 + h = 0.05 + state_dim = 40 + tanl = 0.05 + nanl = 6500 + spin = 1500 + diffusion = 0.00 + F = 8.0 + + # generate truth twin time series + GenerateTimeSeries.L96_time_series( + ( + seed = seed, + h = h, + state_dim = state_dim, + tanl = tanl, + nanl = nanl, + spin = spin, + diffusion = diffusion, + F = F, + ) + ) + + # define load path to time series + time_series = path * "L96_time_series_seed_" * lpad(seed, 4, "0") * + "_dim_" * lpad(state_dim, 2, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_F_" * lpad(F, 4, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_spin_" * lpad(spin, 4, "0") * + "_h_" * rpad(h, 5, "0") * + ".jld2" + + # define ranges for filter parameters + methods = ["enkf-n-primal", "enkf-n-primal-ls", "enkf-n-dual"] + seed = 1234 + obs_un = 1.0 + obs_dim = 40 + N_enss = 15:3:42 + s_infls = [1.0] + nanl = 4000 + γ = 1.0 + + # load the experiments + args = Vector{Any}() + for method in methods + for N_ens in N_enss + for s_infl in s_infls + tmp = ( + time_series = time_series, + method = method, + seed = seed, + nanl = nanl, + obs_un = obs_un, + obs_dim = obs_dim, + γ = γ, + N_ens = N_ens, + s_infl = s_infl + ) + push!(args, tmp) + end + end + end + return args, wrap_exp +end + + ############################################################################################## -# filter_param -## [time_series, scheme, seed, nanl, obs_un, obs_dim, param_err, param_wlk, N_ens, -## state_infl, param_infl] = args -# -#schemes = ["enkf", "etkf"] -#seed = 0 -#obs_un = 1.0 -#obs_dim = 40 -#param_err = 0.03 -#param_wlk = [0.0000, 0.0001, 0.0010, 0.0100] -#N_ens = 14:41 -#state_infl = LinRange(1.0, 1.20, 21) -#param_infl = LinRange(1.0, 1.00, 1) -#nanl = 2500 -# -## load the experiments -#args = Tuple[] -#for scheme in schemes -# for wlk in param_wlk -# for N in N_ens -# for s_infl in state_infl -# for p_infl in param_infl -# tmp = (time_series, scheme, seed, nanl, obs_un, obs_dim, -# param_err, wlk, N, s_infl, p_infl) -# push!(args, tmp) -# end -# end -# end -# end -#end -# -#experiment = FilterExps.filter_param -# -# +# parameter estimation, different random walk and inflation settings for parameter resampling + +""" + args, exp = ensemble_filter_param() + +Constucts a parameter map and experiment wrapper for sensitivity test of parameter estimation. + +Ensemble schemes sample the forcing parameter for the Lorenz-96 system and vary the random +walk parameter model for its time evolution / search over parameter space. +""" +function ensemble_filter_param() + + exp = DataAssimilationBenchmarks.FilterExps.ensemble_filter_param + function wrap_exp(arguments) + try + exp(arguments) + catch + print("Error on " * string(arguments) * "\n") + end + end + + # set time series parameters + seed = 123 + h = 0.05 + state_dim = 40 + tanl = 0.05 + nanl = 7500 + spin = 1500 + diffusion = 0.00 + F = 8.0 + + # generate truth twin time series + GenerateTimeSeries.L96_time_series( + ( + seed = seed, + h = h, + state_dim = state_dim, + tanl = tanl, + nanl = nanl, + spin = spin, + diffusion = diffusion, + F = F, + ) + ) + + # define load path to time series + time_series = path * "L96_time_series_seed_" * lpad(seed, 4, "0") * + "_dim_" * lpad(state_dim, 2, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_F_" * lpad(F, 4, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_spin_" * lpad(spin, 4, "0") * + "_h_" * rpad(h, 5, "0") * + ".jld2" + + # define ranges for filter parameters + methods = ["etkf", "mlef-transform"] + seed = 1234 + obs_un = 1.0 + obs_dim = 40 + p_err = 0.03 + p_wlks = [0.0000, 0.0001, 0.0010, 0.0100] + N_enss = 15:3:42 + s_infls = [1.0] + nanl = 4000 + s_infls = LinRange(1.0, 1.10, 11) + p_infls = LinRange(1.0, 1.05, 6) + γ = 1.0 + + # load the experiments + args = Vector{Any}() + for method in methods + for p_wlk in p_wlks + for N_ens in N_enss + for s_infl in s_infls + for p_infl in p_infls + tmp = ( + time_series = time_series, + method = method, + seed = seed, + nanl = nanl, + obs_un = obs_un, + obs_dim = obs_dim, + γ = γ, + p_err = p_err, + p_wlk = p_wlk, + N_ens = N_ens, + s_infl = s_infl, + p_infl = p_infl + ) + push!(args, tmp) + end + end + end + end + end + return args, wrap_exp +end + + ############################################################################################## # Classic smoothers ############################################################################################## -# classic_state parallel run, arguments are -# time_series, method, seed, nanl, lag, shift, obs_un, obs_dim, γ, N_ens, state_infl = args - -schemes = ["etks"] -seed = 0 -lags = 1:3:52 -shifts = [1] -#lags = [1, 2, 4, 8, 16, 32, 64] -#gammas = Array{Float64}(1:11) -gammas = [1.0] -shift = 1 -obs_un = 1.0 -obs_dim = 40 -#N_ens = 15:2:41 -N_ens = [21] -#state_infl = [1.0] -state_infl = LinRange(1.0, 1.10, 11) -time_series = [ts1, ts2, ts3, ts4, ts5] -nanl = 2500 - -# load the experiments -args = Tuple[] -for ts in time_series - for scheme in schemes + +""" + args, exp = classic_ensemble_state() + +Constucts a parameter map and experiment wrapper for sensitivity test of nonlinear obs. + +The ETKS / MLES estimators vary over different multiplicative inflation parameters, smoother +lag lengths and the nonlinearity of the observation operator. +""" +function classic_ensemble_state() + + exp = DataAssimilationBenchmarks.SmootherExps.classic_ensemble_state + function wrap_exp(arguments) + try + exp(arguments) + catch + print("Error on " * string(arguments) * "\n") + end + end + + # set time series parameters + seed = 123 + h = 0.05 + state_dim = 40 + tanl = 0.05 + nanl = 7500 + spin = 1500 + diffusion = 0.00 + F = 8.0 + + # generate truth twin time series + GenerateTimeSeries.L96_time_series( + ( + seed = seed, + h = h, + state_dim = state_dim, + tanl = tanl, + nanl = nanl, + spin = spin, + diffusion = diffusion, + F = F, + ) + ) + + # define load path to time series + time_series = path * "L96_time_series_seed_" * lpad(seed, 4, "0") * + "_dim_" * lpad(state_dim, 2, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_F_" * lpad(F, 4, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_spin_" * lpad(spin, 4, "0") * + "_h_" * rpad(h, 5, "0") * + ".jld2" + + # define ranges for filter parameters + methods = ["etks", "mles-transform"] + seed = 1234 + lags = 1:3:52 + shifts = [1] + gammas = Vector{Float64}(1:10) + shift = 1 + obs_un = 1.0 + obs_dim = 40 + N_enss = 15:3:42 + s_infls = LinRange(1.0, 1.10, 11) + nanl = 4000 + + # load the experiments + args = Vector{Any}() + for method in methods for γ in gammas - for l in 1:length(lags) - # optional definition of shifts in terms of the current lag parameter for a - # range of shift values - lag = lags[l] - #shifts = lags[1:l] + for lag in lags for shift in shifts - for N in N_ens - for s_infl in state_infl - tmp = (ts, scheme, seed, nanl, lag, shift, obs_un, obs_dim, - γ, N, s_infl) + for N_ens in N_enss + for s_infl in s_infls + tmp = ( + time_series = time_series, + method = method, + seed = seed, + nanl = nanl, + lag = lag, + shift = shift, + obs_un = obs_un, + obs_dim = obs_dim, + γ = γ, + N_ens = N_ens, + s_infl = s_infl + ) push!(args, tmp) end end @@ -143,126 +303,314 @@ for ts in time_series end end end + return args, wrap_exp end +############################################################################################# -## define the robust to failure wrapper -#function wrap_exp(arguments) -# try -# classic_state(arguments) -# catch -# print("Error on " * string(args) * "\n") -# end -#end +""" + args, exp = ensemble_filter_adaptive_inflation() -experiment = SmootherExps.classic_state -#experiment = wrap_exp +Constucts a parameter map and experiment wrapper for sensitivity test of parameter estimation. +Ensemble schemes sample the forcing parameter for the Lorenz-96 system and vary the random +walk parameter model for its time evolution / search over parameter space. Methods vary +the ETKS and MLES analysis, with different lag lengths, multiplicative inflation parameters, +and different pameter models. +""" +function classic_ensemble_param() + + exp = DataAssimilationBenchmarks.SmootherExps.classic_ensemble_param + function wrap_exp(arguments) + try + exp(arguments) + catch + print("Error on " * string(arguments) * "\n") + end + end + + # set time series parameters + seed = 123 + h = 0.05 + state_dim = 40 + tanl = 0.05 + nanl = 7500 + spin = 1500 + diffusion = 0.00 + F = 8.0 + + # generate truth twin time series + GenerateTimeSeries.L96_time_series( + ( + seed = seed, + h = h, + state_dim = state_dim, + tanl = tanl, + nanl = nanl, + spin = spin, + diffusion = diffusion, + F = F, + ) + ) + + # define load path to time series + time_series = path * "L96_time_series_seed_" * lpad(seed, 4, "0") * + "_dim_" * lpad(state_dim, 2, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_F_" * lpad(F, 4, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_spin_" * lpad(spin, 4, "0") * + "_h_" * rpad(h, 5, "0") * + ".jld2" + + # define ranges for filter parameters + methods = ["etks", "mles-transform"] + seed = 1234 + lags = 1:3:52 + shifts = [1] + gammas = [1.0] + shift = 1 + obs_un = 1.0 + obs_dim = 40 + N_enss = 15:3:42 + p_err = 0.03 + p_wlks = [0.0000, 0.0001, 0.0010, 0.0100] + s_infls = LinRange(1.0, 1.10, 11) + p_infls = LinRange(1.0, 1.05, 6) + nanl = 4000 + + # load the experiments + args = Vector{Any}() + for method in methods + for lag in lags + for γ in gammas + for N_ens in N_enss + for p_wlk in p_wlks + for s_infl in s_infls + for p_infl in p_infls + tmp = ( + time_series = time_series, + method = method, + seed = seed, + nanl = nanl, + lag = lag, + shift = shift, + obs_un = obs_un, + obs_dim = obs_dim, + γ = γ, + p_err = p_err, + p_wlk = p_wlk, + N_ens = N_ens, + s_infl = s_infl, + p_infl = p_infl + ) + push!(args, tmp) + end + end + end + end + end + end + end + return args, wrap_exp +end + + +############################################################################################# +# SIEnKS +############################################################################################# + +function single_iteration_ensemble_state() + + exp = DataAssimilationBenchmarks.SmootherExps.single_iteration_ensemble_state + function wrap_exp(arguments) + try + exp(arguments) + catch + print("Error on " * string(arguments) * "\n") + end + end + + # set time series parameters + seed = 123 + h = 0.05 + state_dim = 40 + tanl = 0.05 + nanl = 7500 + spin = 1500 + diffusion = 0.00 + F = 8.0 + + # generate truth twin time series + GenerateTimeSeries.L96_time_series( + ( + seed = seed, + h = h, + state_dim = state_dim, + tanl = tanl, + nanl = nanl, + spin = spin, + diffusion = diffusion, + F = F, + ) + ) + + # define load path to time series + time_series = path * "L96_time_series_seed_" * lpad(seed, 4, "0") * + "_dim_" * lpad(state_dim, 2, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_F_" * lpad(F, 4, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_spin_" * lpad(spin, 4, "0") * + "_h_" * rpad(h, 5, "0") * + ".jld2" + + # define ranges for filter parameters + methods = ["etks"] + seed = 1234 + lags = 1:3:52 + shifts = [1] + gammas = [1.0] + shift = 1 + obs_un = 1.0 + obs_dim = 40 + N_enss = 15:3:42 + s_infls = LinRange(1.0, 1.10, 11) + nanl = 4000 + mdas = [false, true] + + # load the experiments + args = Vector{Any}() + for mda in mdas + for γ in gammas + for method in methods + for lag in lags + for shift in shifts + for N_ens in N_enss + for s_infl in s_infls + tmp = ( + time_series = time_series, + method = method, + seed = seed, + nanl = nanl, + lag = lag, + shift = shift, + mda = mda, + obs_un = obs_un, + obs_dim = obs_dim, + γ = γ, + N_ens = N_ens, + s_infl = s_infl + ) + push!(args, tmp) + end + end + end + end + end + end + end + return args, wrap_exp +end + + +############################################################################################# +# IEnKS +############################################################################################# + +function iterative_ensemble_state() + + exp = DataAssimilationBenchmarks.SmootherExps.iterative_ensemble_state + function wrap_exp(arguments) + try + exp(arguments) + catch + print("Error on " * string(arguments) * "\n") + end + end + + # set time series parameters + seed = 123 + h = 0.05 + state_dim = 40 + tanl = 0.05 + nanl = 7500 + spin = 1500 + diffusion = 0.00 + F = 8.0 + + # generate truth twin time series + GenerateTimeSeries.L96_time_series( + ( + seed = seed, + h = h, + state_dim = state_dim, + tanl = tanl, + nanl = nanl, + spin = spin, + diffusion = diffusion, + F = F, + ) + ) + + # define load path to time series + time_series = path * "L96_time_series_seed_" * lpad(seed, 4, "0") * + "_dim_" * lpad(state_dim, 2, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_F_" * lpad(F, 4, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_spin_" * lpad(spin, 4, "0") * + "_h_" * rpad(h, 5, "0") * + ".jld2" + + # define ranges for filter parameters + methods = ["ienks-transform", "lin-ienks-transform"] + seed = 1234 + lags = 1:3:52 + gammas = [1.0] + shift = 1 + obs_un = 1.0 + obs_dim = 40 + N_enss = 15:3:42 + s_infls = LinRange(1.0, 1.10, 11) + nanl = 4000 + mdas = [false, true] + + # load the experiments + args = Vector{Any}() + for mda in mdas + for γ in gammas + for method in methods + for lag in lags + for N_ens in N_enss + for s_infl in s_infls + tmp = ( + time_series = time_series, + method = method, + seed = seed, + nanl = nanl, + lag = lag, + shift = shift, + mda = mda, + obs_un = obs_un, + obs_dim = obs_dim, + γ = γ, + N_ens = N_ens, + s_infl = s_infl + ) + push!(args, tmp) + end + end + end + end + end + end + return args, exp +end -############################################################################################## -## classic_param single run for debugging, arguments are -## [time_series, method, seed, nanl, lag, shift, obs_un, obs_dim, param_err, -## param_wlk, N_ens, state_infl, param_infl = args -# -#schemes = ["enks", "etks"] -#seed = 0 -#lag = 1:5:51 -#shift = 1 -#obs_un = 1.0 -#obs_dim = 40 -#N_ens = 14:41 -#param_err = 0.03 -#param_wlk = [0.0000, 0.0001, 0.0010, 0.0100] -#state_infl = LinRange(1.0, 1.20, 21) -#param_infl = LinRange(1.0, 1.00, 1) -#nanl = 2500 -# -## load the experiments -#args = Tuple[] -#for scheme in schemes -# for l in lag -# for N in N_ens -# for wlk in param_wlk -# for s_infl in state_infl -# for p_infl in param_infl -# tmp = (time_series, scheme, seed, nanl, l, shift, obs_un, obs_dim, -# param_err, wlk, N, s_infl, p_infl) -# push!(args, tmp) -# end -# end -# end -# end -# end -#end -# -#experiment = SmootherExps.classic_param -# -# -############################################################################################## -# Single iteration smoothers -############################################################################################## -## single iteration single run for degbugging, arguments are -## [time_series, method, seed, nanl, lag, shift, mda, obs_un, obs_dim, -## N_ens, state_infl = args -# -#schemes = ["enks-n-primal"] -#seed = 0 -##lags = 1:3:52 -#lags = [1, 2, 4, 8, 16, 32, 64] -##gammas = Array{Float64}(1:11) -#gammas = [1.0] -##shift = 1 -#obs_un = 1.0 -#obs_dim = 40 -##N_ens = 15:2:41 -#N_ens = [21] -#state_infl = [1.0] -##state_infl = LinRange(1.0, 1.10, 11) -#time_series = [time_series_1, time_series_2] -#mdas = [false] -#nanl = 2500 -# -## load the experiments -#args = Tuple[] -#for m in mdas -# for ts in time_series -# for γ in gammas -# for scheme in schemes -# for l in 1:length(lags) -# # optional definition of shift in terms of the current lag parameter -# # for a range of shift values -# lag = lags[l] -# shifts = lags[1:l] -# for shift in shifts -# for N in N_ens -# for s_infl in state_infl -# tmp = (ts, scheme, seed, nanl, lag, shift, m, obs_un, -# obs_dim, γ, N, s_infl) -# push!(args, tmp) -# end -# end -# end -# end -# end -# end -# end -#end -# -## define the robust to failure wrapper -#function wrap_exp(arguments) -# try -# SmootherExps.single_iteration_state(arguments) -# catch -# print("Error on " * string(args) * "\n") -# end -#end -# -#experiment = wrap_exp -# -############################################################################################## -# Run the experiments in parallel over the parameter values -############################################################################################## -pmap(experiment, args) ############################################################################################## # end module diff --git a/src/experiments/SmootherExps.jl b/src/experiments/SmootherExps.jl index fec79289..8fface9f 100755 --- a/src/experiments/SmootherExps.jl +++ b/src/experiments/SmootherExps.jl @@ -18,11 +18,10 @@ export classic_ensemble_state, classic_ensemble_param, single_iteration_ensemble lag::Int64, shift::Int64, obs_un::Float64, obs_dim::Int64, γ::Float64, N_ens::Int64, s_infl::Float64)::NamedTuple) -Classic ensemble Kalman smoother state estimation twin experiment. Twin experiment parameters -such as the observation dimension, observation uncertainty, data assimilation method, number -of cycles, ensemble size etc. are specified in the arguments. NOTE: the classic scheme does -not use multiple data assimilation and we hard code `mda=false` in the function for -consistency with the API of other methods. +Classic ensemble Kalman smoother state estimation twin experiment. + +NOTE: the classic scheme does not use multiple data assimilation and we hard code +`mda=false` in the function for consistency with the API of other methods. Output from the experiment is saved in a dictionary of the form, @@ -57,7 +56,7 @@ Output from the experiment is saved in a dictionary of the form, Experiment output is written to a directory defined by - pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "-classic/" + path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "-classic/" where the file name is written dynamically according to the selected parameters as follows: @@ -299,11 +298,71 @@ end γ::Float64, p_err::Float64, p_wlk::Float64, N_ens::Int64, s_infl::Float64, s_infl::Float64})::NamedTuple) -Classic ensemble Kalman smoother joint state-parameter estimation twin experiment. Twin -experiment parameters such as the observation dimension, observation uncertainty, data -assimilation method, number of cycles, ensemble size etc. are specified in the arguments. +Classic ensemble Kalman smoother joint state-parameter estimation twin experiment. + NOTE: the classic scheme does not use multiple data assimilation and we hard code `mda=false` in the function for consistency with other methods. + +Output from the experiment is saved in a dictionary of the form, + + data = Dict{String,Any}( + "fore_rmse" => fore_rmse, + "filt_rmse" => filt_rmse, + "post_rmse" => post_rmse, + "param_rmse" => para_rmse, + "fore_spread" => fore_spread, + "filt_spread" => filt_spread, + "post_spread" => post_spread, + "param_spread" => para_spread, + "method" => method, + "seed" => seed, + "diffusion" => diffusion, + "dx_params" => dx_params, + "param_truth" => param_truth, + "sys_dim" => sys_dim, + "state_dim" => state_dim, + "obs_dim" => obs_dim, + "obs_un" => obs_un, + "gamma" => γ, + "p_err" => p_err, + "p_wlk" => p_wlk, + "nanl" => nanl, + "tanl" => tanl, + "lag" => lag, + "shift" => shift, + "mda" => mda, + "h" => h, + "N_ens" => N_ens, + "s_infl" => round(s_infl, digits=2), + "p_infl" => round(p_infl, digits=2) + ) + + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"] + end + +Experiment output is written to a directory defined by + + path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "-classic/" + +where the file name is written dynamically according to the selected parameters as follows: + + method * "-classic_" * model * + "_state_seed_" * lpad(seed, 4, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_sysD_" * lpad(sys_dim, 2, "0") * + "_obsD_" * lpad(obs_dim, 2, "0") * + "_obsU_" * rpad(obs_un, 4, "0") * + "_gamma_" * lpad(γ, 5, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_h_" * rpad(h, 4, "0") * + "_lag_" * lpad(lag, 3, "0") * + "_shift_" * lpad(shift, 3, "0") * + "_mda_" * string(mda) * + "_nens_" * lpad(N_ens, 3,"0") * + "_stateInfl_" * rpad(round(s_infl, digits=2), 4, "0") * + ".jld2" """ function classic_ensemble_param((time_series, method, seed, nanl, lag, shift, obs_un, obs_dim, γ, p_err, p_wlk, N_ens, s_infl, p_infl)::NamedTuple{ @@ -543,6 +602,10 @@ function classic_ensemble_param((time_series, method, seed, nanl, lag, shift, ob "p_infl" => round(p_infl, digits=2) ) + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"] + end + path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "-classic/" name = method * "-classic_" * model * "_param_seed_" * lpad(seed, 4, "0") * @@ -577,9 +640,61 @@ end obs_un::Float64, obs_dim::Int64, γ::Float64, N_ens::Int64, s_infl::Float64})::NamedTuple) -SIEnKS state estimation twin experiment. Twin experiment parameters -such as the observation dimension, observation uncertainty, data assimilation method, number -of cycles, ensemble size etc. are specified in the arguments. +SIEnKS state estimation twin experiment. + +Output from the experiment is saved in a dictionary of the form, + + data = Dict{String,Any}( + "fore_rmse" => fore_rmse, + "filt_rmse" => filt_rmse, + "post_rmse" => post_rmse, + "fore_spread" => fore_spread, + "filt_spread" => filt_spread, + "post_spread" => post_spread, + "method" => method, + "seed" => seed, + "diffusion" => diffusion, + "dx_params" => dx_params, + "sys_dim" => sys_dim, + "obs_dim" => obs_dim, + "obs_un" => obs_un, + "gamma" => γ, + "nanl" => nanl, + "tanl" => tanl, + "lag" => lag, + "shift" => shift, + "mda" => mda, + "h" => h, + "N_ens" => N_ens, + "s_infl" => round(s_infl, digits=2) + ) + + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"] + end + +Experiment output is written to a directory defined by + + path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "-single-iteration/" + +where the file name is written dynamically according to the selected parameters as follows: + + method * "-single-iteration_" * model * + "_state_seed_" * lpad(seed, 4, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_sysD_" * lpad(sys_dim, 2, "0") * + "_obsD_" * lpad(obs_dim, 2, "0") * + "_obsU_" * rpad(obs_un, 4, "0") * + "_gamma_" * lpad(γ, 5, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_h_" * rpad(h, 4, "0") * + "_lag_" * lpad(lag, 3, "0") * + "_shift_" * lpad(shift, 3, "0") * + "_mda_" * string(mda) * + "_nens_" * lpad(N_ens, 3,"0") * + "_stateInfl_" * rpad(round(s_infl, digits=2), 4, "0") * + ".jld2" """ function single_iteration_ensemble_state((time_series, method, seed, nanl, lag, shift, mda, obs_un, obs_dim, γ, N_ens, s_infl)::NamedTuple{ @@ -883,9 +998,70 @@ end p_err::Float64, p_wlk::Float64, N_ens::Int64, s_infl::Float64, p_infl::Float64)::NamedTuple) -SIEnKS joint state-parameter estimation twin experiment. Twin experiment parameters -such as the observation dimension, observation uncertainty, data assimilation method, number -of cycles, ensemble size etc. are specified in the arguments. +SIEnKS joint state-parameter estimation twin experiment. + +Output from the experiment is saved in a dictionary of the form, + + data = Dict{String,Any}( + "fore_rmse" => fore_rmse, + "filt_rmse" => filt_rmse, + "post_rmse" => post_rmse, + "param_rmse" => para_rmse, + "fore_spread" => fore_spread, + "filt_spread" => filt_spread, + "post_spread" => post_spread, + "param_spread" => para_spread, + "method" => method, + "seed" => seed, + "diffusion" => diffusion, + "dx_params" => dx_params, + "param_truth" => param_truth, + "sys_dim" => sys_dim, + "obs_dim" => obs_dim, + "obs_un" => obs_un, + "gamma" => γ, + "p_wlk" => p_wlk, + "p_infl" => p_infl, + "nanl" => nanl, + "tanl" => tanl, + "lag" => lag, + "shift" => shift, + "mda" => mda, + "h" => h, + "N_ens" => N_ens, + "s_infl" => round(s_infl, digits=2), + "p_infl" => round(p_infl, digits=2) + ) + + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"] + end + +Experiment output is written to a directory defined by + + path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "-single-iteration/" + +where the file name is written dynamically according to the selected parameters as follows: + + method * "-single-iteration_" * model * + "_param_seed_" * lpad(seed, 4, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_sysD_" * lpad(sys_dim, 2, "0") * + "_obsD_" * lpad(obs_dim, 2, "0") * + "_obsU_" * rpad(obs_un, 4, "0") * + "_gamma_" * lpad(γ, 5, "0") * + "_paramE_" * rpad(p_err, 4, "0") * + "_paramW_" * rpad(p_wlk, 6, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_h_" * rpad(h, 4, "0") * + "_lag_" * lpad(lag, 3, "0") * + "_shift_" * lpad(shift, 3, "0") * + "_mda_" * string(mda) * + "_nens_" * lpad(N_ens, 3,"0") * + "_stateInfl_" * rpad(round(s_infl, digits=2), 4, "0") * + "_paramInfl_" * rpad(round(p_infl, digits=2), 4, "0") * + ".jld2" """ function single_iteration_ensemble_param((time_series, method, seed, nanl, lag, shift, mda, obs_un, obs_dim, γ, p_err, p_wlk, N_ens, s_infl, @@ -1213,6 +1389,9 @@ function single_iteration_ensemble_param((time_series, method, seed, nanl, lag, "p_infl" => round(p_infl, digits=2) ) + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"] + end path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "-single-iteration/" name = method * "-single-iteration_" * model * @@ -1249,9 +1428,62 @@ end obs_dim::Int64, γ::Float64, N_ens::Int64, s_infl::Float64)::NamedTuple) -4DEnVAR state estimation. Twin experiment parameters -such as the observation dimension, observation uncertainty, data assimilation method, number -of cycles, ensemble size etc. are specified in the arguments. +4DEnVAR state estimation twin experiment using the IEnKS formalism. + +Output from the experiment is saved in a dictionary of the form, + + data = Dict{String,Any}( + "fore_rmse" => fore_rmse, + "filt_rmse" => filt_rmse, + "post_rmse" => post_rmse, + "fore_spread" => fore_spread, + "filt_spread" => filt_spread, + "post_spread" => post_spread, + "iteration_sequence" => iteration_sequence, + "method" => method, + "seed" => seed, + "diffusion" => diffusion, + "dx_params" => dx_params, + "sys_dim" => sys_dim, + "obs_dim" => obs_dim, + "obs_un" => obs_un, + "gamma" => γ, + "nanl" => nanl, + "tanl" => tanl, + "lag" => lag, + "shift" => shift, + "mda" => mda, + "h" => h, + "N_ens" => N_ens, + "s_infl" => round(s_infl, digits=2) + ) + + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"] + end + +Experiment output is written to a directory defined by + + path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "/" + +where the file name is written dynamically according to the selected parameters as follows: + + method * "_" * model * + "_state_seed_" * lpad(seed, 4, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_sysD_" * lpad(sys_dim, 2, "0") * + "_obsD_" * lpad(obs_dim, 2, "0") * + "_obsU_" * rpad(obs_un, 4, "0") * + "_gamma_" * lpad(γ, 5, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_h_" * rpad(h, 4, "0") * + "_lag_" * lpad(lag, 3, "0") * + "_shift_" * lpad(shift, 3, "0") * + "_mda_" * string(mda) * + "_nens_" * lpad(N_ens, 3,"0") * + "_stateInfl_" * rpad(round(s_infl, digits=2), 4, "0") * + ".jld2" """ function iterative_ensemble_state((time_series, method, seed, nanl, lag, shift, mda, obs_un, obs_dim,γ, N_ens, s_infl)::NamedTuple{ @@ -1574,7 +1806,6 @@ function iterative_ensemble_state((time_series, method, seed, nanl, lag, shift, "_stateInfl_" * rpad(round(s_infl, digits=2), 4, "0") * ".jld2" - save(path * name, data) print("Runtime " * string(round((time() - t1) / 60.0, digits=4)) * " minutes\n") @@ -1588,9 +1819,70 @@ end obs_dim::Int64, γ::Float64, p_err::Float64, p_wlk::Float64, N_ens::Int64, s_infl::Float64, p_infl::Float64)::NamedTuple) -4DEnVAR joint state-parameter estimation twin experiment. Twin experiment parameters -such as the observation dimension, observation uncertainty, data assimilation method, number -of cycles, ensemble size etc. are specified in the arguments. +4DEnVAR joint state-parameter estimation twin experiment using the IEnKS formalism. + +Output from the experiment is saved in a dictionary of the form, + + data = Dict{String,Any}( + "fore_rmse" => fore_rmse, + "filt_rmse" => filt_rmse, + "post_rmse" => post_rmse, + "param_rmse" => para_rmse, + "fore_spread" => fore_spread, + "filt_spread" => filt_spread, + "post_spread" => post_spread, + "param_spread" => para_spread, + "iteration_sequence" => iteration_sequence, + "method" => method, + "seed" => seed, + "diffusion" => diffusion, + "dx_params" => dx_params, + "sys_dim" => sys_dim, + "obs_dim" => obs_dim, + "obs_un" => obs_un, + "gamma" => γ, + "p_wlk" => p_wlk, + "p_infl" => p_infl, + "nanl" => nanl, + "tanl" => tanl, + "lag" => lag, + "shift" => shift, + "mda" => mda, + "h" => h, + "N_ens" => N_ens, + "s_infl" => round(s_infl, digits=2), + "p_infl" => round(p_infl, digits=2) + ) + + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"] + end + +Experiment output is written to a directory defined by + + path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "/" + +where the file name is written dynamically according to the selected parameters as follows: + + method * "_" * model * + "_param_seed_" * lpad(seed, 4, "0") * + "_diff_" * rpad(diffusion, 5, "0") * + "_sysD_" * lpad(sys_dim, 2, "0") * + "_obsD_" * lpad(obs_dim, 2, "0") * + "_obsU_" * rpad(obs_un, 4, "0") * + "_gamma_" * lpad(γ, 5, "0") * + "_paramE_" * rpad(p_err, 4, "0") * + "_paramW_" * rpad(p_wlk, 6, "0") * + "_nanl_" * lpad(nanl, 5, "0") * + "_tanl_" * rpad(tanl, 4, "0") * + "_h_" * rpad(h, 4, "0") * + "_lag_" * lpad(lag, 3, "0") * + "_shift_" * lpad(shift, 3, "0") * + "_mda_" * string(mda) * + "_nens_" * lpad(N_ens, 3,"0") * + "_stateInfl_" * rpad(round(s_infl, digits=2), 4, "0") * + "_paramInfl_" * rpad(round(p_infl, digits=2), 4, "0") * + ".jld2" """ function iterative_ensemble_param((time_series, method, seed, nanl, lag, shift, mda, obs_un, obs_dim, γ, p_err, p_wlk, N_ens, s_infl, @@ -1925,6 +2217,10 @@ function iterative_ensemble_param((time_series, method, seed, nanl, lag, shift, "p_infl" => round(p_infl, digits=2) ) + if haskey(ts, "diff_mat") + data["diff_mat"] = ts["diff_mat"] + end + path = pkgdir(DataAssimilationBenchmarks) * "/src/data/" * method * "/" name = method * "_" * model * "_param_seed_" * lpad(seed, 4, "0") * @@ -1946,7 +2242,6 @@ function iterative_ensemble_param((time_series, method, seed, nanl, lag, shift, "_paramInfl_" * rpad(round(p_infl, digits=2), 4, "0") * ".jld2" - save(path * name, data) print("Runtime " * string(round((time() - t1) / 60.0, digits=4)) * " minutes\n") end diff --git a/src/experiments/run_sensitivity_test.jl b/src/experiments/run_sensitivity_test.jl new file mode 100644 index 00000000..98a9ffe2 --- /dev/null +++ b/src/experiments/run_sensitivity_test.jl @@ -0,0 +1,30 @@ +############################################################################################## +module run_sensitivity_test +############################################################################################## +# imports and exports +using Distributed +@everywhere using DataAssimilationBenchmarks + +############################################################################################## + +config = ParallelExperimentDriver.ensemble_filter_adaptive_inflation + +print("Generating experiment configurations from " * string(config) * "\n") +print("Generate truth twin\n") + +args, exp = config() +num_exps = length(args) + +print("Configuration ready\n") +print("\n") +print("Running " * string(num_exps) * " configurations on " * string(nworkers()) * + " total workers\n") +print("Begin pmap\n") +pmap(exp, args) +print("Experiments completed, verify outputs in the appropriate directory under:\n") +print(pkgdir(DataAssimilationBenchmarks) * "/src/data\n") + +############################################################################################## +# end module + +end