From bb5ed5cfe0cfbd6939ad6af2980412178d3cf634 Mon Sep 17 00:00:00 2001 From: jwhite Date: Fri, 19 Apr 2024 10:20:15 -0600 Subject: [PATCH 01/27] fix in mou to save populations on last generation always --- src/libs/pestpp_common/MOEA.cpp | 7 +++++-- src/libs/pestpp_common/MOEA.h | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libs/pestpp_common/MOEA.cpp b/src/libs/pestpp_common/MOEA.cpp index 9f825f16..e455ae81 100644 --- a/src/libs/pestpp_common/MOEA.cpp +++ b/src/libs/pestpp_common/MOEA.cpp @@ -2993,6 +2993,9 @@ void MOEA::iterate_to_solution() if ((q == 1) || (q == 2)) { message(0,"'pest.stp' found, quitting"); + message(1,"force-saving current populations"); + save_populations(dp,op,"",true); + save_populations(dp_archive, op_archive, "archive",true); break; } else if (q == 4) { @@ -4126,7 +4129,7 @@ ParameterEnsemble MOEA::generate_sbx_population(int num_members, ParameterEnsemb return tmp_dp; } -void MOEA::save_populations(ParameterEnsemble& _dp, ObservationEnsemble& _op, string tag) +void MOEA::save_populations(ParameterEnsemble& _dp, ObservationEnsemble& _op, string tag, bool force_save) { stringstream ss; @@ -4153,7 +4156,7 @@ void MOEA::save_populations(ParameterEnsemble& _dp, ObservationEnsemble& _op, st ss << "saved decision variable population of size " << _dp.shape().first << " X " << _dp.shape().second << " to '" << name << "'"; message(1, ss.str()); ss.str(""); - if ((save_every > 0) && (iter % save_every == 0)) + if (((save_every > 0) && (iter % save_every == 0)) || (iter == pest_scenario.get_control_info().noptmax) || (force_save)) { ss << file_manager.get_base_filename() << "." << iter; if (tag.size() > 0) diff --git a/src/libs/pestpp_common/MOEA.h b/src/libs/pestpp_common/MOEA.h index cbecfdc3..265f1bb4 100644 --- a/src/libs/pestpp_common/MOEA.h +++ b/src/libs/pestpp_common/MOEA.h @@ -224,7 +224,7 @@ class MOEA string get_new_member_name(string tag = string()); - void save_populations(ParameterEnsemble& _dp, ObservationEnsemble& _op, string tag = string()); + void save_populations(ParameterEnsemble& _dp, ObservationEnsemble& _op, string tag = string(), bool force_save=false); void gauss_mutation_ip(ParameterEnsemble& _dp); pair sbx(double probability, double eta_m, int idx1, int idx2); pair sbx_new(double crossover_probability, double di, Eigen::VectorXd& parent1, From 87f466dc17e70b84101b1e29cb69c53b55357bf1 Mon Sep 17 00:00:00 2001 From: jwhite Date: Sun, 21 Apr 2024 08:13:37 -0600 Subject: [PATCH 02/27] trying to speed up feasible sorting --- src/libs/pestpp_common/MOEA.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libs/pestpp_common/MOEA.cpp b/src/libs/pestpp_common/MOEA.cpp index e455ae81..753017da 100644 --- a/src/libs/pestpp_common/MOEA.cpp +++ b/src/libs/pestpp_common/MOEA.cpp @@ -291,9 +291,11 @@ void ParetoObjectives::update(ObservationEnsemble& op, ParameterEnsemble& dp, Co vector onames = op.get_var_names(), pnames = dp.get_var_names(); set obs_obj_set(obs_obj_names_ptr->begin(), obs_obj_names_ptr->end()); set pi_obj_set(pi_obj_names_ptr->begin(), pi_obj_names_ptr->end()); + set::iterator end; ObservationInfo *oi = pest_scenario.get_observation_info_ptr(); PriorInformation *pi = pest_scenario.get_prior_info_ptr(); feas_member_struct.clear(); + for (auto real_name : real_names) { vsum = 0.0; obs.update_without_clear(onames, op.get_real_vector(real_name)); @@ -301,14 +303,16 @@ void ParetoObjectives::update(ObservationEnsemble& op, ParameterEnsemble& dp, Co // the 'false' arg is to not apply risk shifting to the satisfaction calcs since // 'op' has already been shifted violations = constraints_ptr->get_unsatified_obs_constraints(obs, 0.0, false); + end = obs_obj_set.end()=; for (auto v : violations) { - if (obs_obj_set.find(v.first) == obs_obj_set.end()) + if (obs_obj_set.find(v.first) == end) vsum += pow(v.second * oi->get_weight(v.first), 2); } pars.update_without_clear(pnames, dp.get_real_vector(real_name)); violations = constraints_ptr->get_unsatified_pi_constraints(pars, 0.0); + end = pi_obj_set.end(); for (auto v : violations) { - if (pi_obj_set.find(v.first) == pi_obj_set.end()) + if (pi_obj_set.find(v.first) == end) vsum += pow(v.second * pi->get_pi_rec(v.first).get_weight(), 2); } if (vsum > 0.0) { From 7cc32d23fb10f395c3342e85176bc0d80773de74 Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 22 Apr 2024 15:17:27 -0600 Subject: [PATCH 03/27] fix in ParameterEnsemble::zeros_like() --- src/libs/pestpp_common/Ensemble.cpp | 12 ++++++++++-- src/libs/pestpp_common/MOEA.cpp | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libs/pestpp_common/Ensemble.cpp b/src/libs/pestpp_common/Ensemble.cpp index 6f9b97f4..0701f1e0 100644 --- a/src/libs/pestpp_common/Ensemble.cpp +++ b/src/libs/pestpp_common/Ensemble.cpp @@ -2166,10 +2166,18 @@ ParameterEnsemble ParameterEnsemble::zeros_like(int nrows) if (nrows < 0) nrows = real_names.size(); Eigen::MatrixXd new_reals = Eigen::MatrixXd::Zero(nrows, var_names.size()); - + stringstream ss; vector new_real_names; for (int i = 0; i < nrows; i++) - new_real_names.push_back(real_names[i]); + if (i < real_names.size()) + new_real_names.push_back(real_names[i]); + else + { + ss.str(""); + ss << "zeros_like_real_" << i; + new_real_names.push_back(ss.str()); + } + ParameterEnsemble new_en(pest_scenario_ptr, rand_gen_ptr); new_en.from_eigen_mat(new_reals, new_real_names, var_names); diff --git a/src/libs/pestpp_common/MOEA.cpp b/src/libs/pestpp_common/MOEA.cpp index 753017da..9db7a5b2 100644 --- a/src/libs/pestpp_common/MOEA.cpp +++ b/src/libs/pestpp_common/MOEA.cpp @@ -303,7 +303,7 @@ void ParetoObjectives::update(ObservationEnsemble& op, ParameterEnsemble& dp, Co // the 'false' arg is to not apply risk shifting to the satisfaction calcs since // 'op' has already been shifted violations = constraints_ptr->get_unsatified_obs_constraints(obs, 0.0, false); - end = obs_obj_set.end()=; + end = obs_obj_set.end(); for (auto v : violations) { if (obs_obj_set.find(v.first) == end) vsum += pow(v.second * oi->get_weight(v.first), 2); From 27a2b6b2b565d82bd846f50be352ad803d34256b Mon Sep 17 00:00:00 2001 From: jwhite Date: Wed, 24 Apr 2024 09:51:16 -0600 Subject: [PATCH 04/27] initial -999 phi factor imp --- .../pestpp_common/EnsembleMethodUtils.cpp | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/libs/pestpp_common/EnsembleMethodUtils.cpp b/src/libs/pestpp_common/EnsembleMethodUtils.cpp index fe1181c2..ea171736 100644 --- a/src/libs/pestpp_common/EnsembleMethodUtils.cpp +++ b/src/libs/pestpp_common/EnsembleMethodUtils.cpp @@ -6456,7 +6456,14 @@ void EnsembleMethod::check_and_fill_phi_factors(map>& grou { for (auto& entry : row.second) { - if (entry.second <= 0.0) + if (entry.second == -999.) + { + ss.str(""); + ss << "phi factor tag '" << row.first << "' has -999 value, indicating that weights for this tag should not be adjusted"; + message(1,ss.str()); + + } + else if (entry.second <= 0.0) { problems.push_back(make_pair(row.first,entry.first)); } @@ -6585,7 +6592,13 @@ void EnsembleMethod::check_and_fill_phi_factors(map>& grou for (auto &g : group_map.at(pf.first)) ss << g << ","; message(2, ss.str()); - if (pf.second <= 0.0) { + if (pf.second == -999.0) + { + ss.str(""); + ss << "phi factor tag '" << pf.first << "' has -999 value, indicating that weights for this tag should not be adjusted"; + message(1,ss.str()); + } + else if (pf.second <= 0.0) { ss.str(""); ss << "adjust_weights(): phi factor '" << pf.first << "' less or equal 0.0 - this is not allowed"; throw_em_error(ss.str()); @@ -6691,6 +6704,13 @@ void EnsembleMethod::adjust_weights_by_real(map>& group_to sub_total=0; scale_fac = 0; for (auto& pf: phi_fracs) { + if (pf.second == -999) + { + ss.str(""); + ss << "NOTE: adjust_weights(): tag " << pf.first << " for realization " << swr_map.first << " has a factor value of -999, skipping this tag"; + message(1,ss.str()); + continue; + } total = 0; for (auto &g : group_map.at(pf.first)) { sub_total = 0; @@ -6822,6 +6842,13 @@ void EnsembleMethod::adjust_weights_single(map>& group_to_ double sub_total = 0; for (auto& pf: phi_fracs) { + if (pf.second == -999) + { + ss.str(""); + ss << "NOTE: adjust_weights(): tag " << pf.first << " has a factor value of -999, skipping this tag"; + message(1,ss.str()); + continue; + } total = 0; for (auto& g : group_map.at(pf.first)) { From 8f71b5c53bff1ab9c840d38319c29c05bf0eb204 Mon Sep 17 00:00:00 2001 From: jwhite Date: Wed, 24 Apr 2024 09:51:52 -0600 Subject: [PATCH 05/27] version --- src/libs/common/config_os.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/common/config_os.h b/src/libs/common/config_os.h index 15d9c7d4..eb876fd0 100644 --- a/src/libs/common/config_os.h +++ b/src/libs/common/config_os.h @@ -2,7 +2,7 @@ #define CONFIG_OS_H_ -#define PESTPP_VERSION "5.2.10"; +#define PESTPP_VERSION "5.2.11"; #if defined(_WIN32) || defined(_WIN64) #define OS_WIN From 2a9de0f75d4ca005b1dee1a5afc230ac4d2040d3 Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 14:18:44 -0600 Subject: [PATCH 06/27] boneheaded fix in reading dense binary --- src/libs/common/utilities.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libs/common/utilities.cpp b/src/libs/common/utilities.cpp index 222878f7..3f716824 100644 --- a/src/libs/common/utilities.cpp +++ b/src/libs/common/utilities.cpp @@ -1127,12 +1127,14 @@ void read_dense_binary(const string& filename, vector& row_names, vector cout << "reading 'dense' format matrix with " << n_obs_and_pi << " columns" << endl; //first read the names of the columns and the rows col_names = read_dense_binary_col_names(in,n_obs_and_pi); + streampos first_record = in.tellg(); row_names = read_dense_binary_remaining_row_names(in,col_names); in.close(); - in.open(filename.c_str(), ifstream::binary); + in.open(filename.c_str(), ifstream::binary); + in.seekg(first_record); //resize the matrix now that we know big it should be matrix.resize(row_names.size(), col_names.size()); From 0fa73ce2a368b55d5e49e617a810b52dd6bc62ee Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 17:12:28 -0600 Subject: [PATCH 07/27] temp revert of error to see if gha will detect a failed test --- .github/workflows/ci.yml | 4 ++-- src/libs/common/utilities.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f38c62d4..a3bf3864 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-latest] - python-version: [3.9] # , 3.7, 3.6] + python-version: [3.10] # , 3.7, 3.6] run-type: [std] # test_repo: [""] # test_dir: [""] @@ -46,7 +46,7 @@ jobs: steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 # - name: Setup Ninja # if: ${{ runner.os == 'Windows' }} - uses: seanmiddleditch/gha-setup-ninja@master diff --git a/src/libs/common/utilities.cpp b/src/libs/common/utilities.cpp index 3f716824..56922790 100644 --- a/src/libs/common/utilities.cpp +++ b/src/libs/common/utilities.cpp @@ -1134,7 +1134,7 @@ void read_dense_binary(const string& filename, vector& row_names, vector in.close(); in.open(filename.c_str(), ifstream::binary); - in.seekg(first_record); + //in.seekg(first_record); //resize the matrix now that we know big it should be matrix.resize(row_names.size(), col_names.size()); From 3c85f394677fefecf10bd7044dc185d75bc485b7 Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 18:14:52 -0600 Subject: [PATCH 08/27] temp revert of error to see if gha will detect a failed test --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3bf3864..7f6f7d30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,13 +174,13 @@ jobs: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: run test script ${{ env.test_script }} in ${{ env.test_dir }} - shell: bash -l {0} + shell: bash run: | ls -l cd ${{ env.test_dir }} nosetests --nocapture -v ${{ env.test_script }} - cd + cd .. From f91031a7565a2c49a9c6924b43caba6b4bc9d271 Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 18:16:47 -0600 Subject: [PATCH 09/27] temp revert of error to see if gha will detect a failed test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f6f7d30..2c014f4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,7 +179,7 @@ jobs: ls -l cd ${{ env.test_dir }} - nosetests --nocapture -v ${{ env.test_script }} + nosetests -v ${{ env.test_script }} cd .. From b439006181488e586c349cdb301d43ba5a57426a Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 18:17:35 -0600 Subject: [PATCH 10/27] more trying to fail a test --- benchmarks/basic_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/basic_tests.py b/benchmarks/basic_tests.py index a9eec28e..93720416 100644 --- a/benchmarks/basic_tests.py +++ b/benchmarks/basic_tests.py @@ -1485,6 +1485,7 @@ def sweep_bin_test(): if __name__ == "__main__": #run() + exit(1) #mf6_v5_ies_test() #prep_ends() sweep_bin_test() From bef876a1d1faecf216a45c77dea8746583fc9815 Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 19:06:36 -0600 Subject: [PATCH 11/27] more trying to fail a test --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c014f4b..9330b9d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,13 +174,13 @@ jobs: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: run test script ${{ env.test_script }} in ${{ env.test_dir }} - shell: bash + shell: bash -l {0} run: | ls -l cd ${{ env.test_dir }} nosetests -v ${{ env.test_script }} - cd .. + cd From 8b496fa7898264d4e06cc59f3b4a62c9cce87336 Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 19:27:26 -0600 Subject: [PATCH 12/27] more CI BS --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9330b9d2..ff63abcc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,9 +174,7 @@ jobs: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: run test script ${{ env.test_script }} in ${{ env.test_dir }} - shell: bash -l {0} - run: | - + run: | ls -l cd ${{ env.test_dir }} nosetests -v ${{ env.test_script }} From 25f498e364177beed6a71c2df7a9680c58798a34 Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 19:30:09 -0600 Subject: [PATCH 13/27] quick fail basic tests --- benchmarks/basic_tests.py | 2837 +++++++++++++++++++------------------ 1 file changed, 1419 insertions(+), 1418 deletions(-) diff --git a/benchmarks/basic_tests.py b/benchmarks/basic_tests.py index 93720416..5ed3d838 100644 --- a/benchmarks/basic_tests.py +++ b/benchmarks/basic_tests.py @@ -1,1494 +1,1495 @@ -import os -import sys -import shutil -import platform -import numpy as np -import pandas as pd -import platform -import pyemu - -bin_path = os.path.join("test_bin") -if "linux" in platform.platform().lower(): - bin_path = os.path.join(bin_path,"linux") -elif "darwin" in platform.platform().lower() or "macos" in platform.platform().lower() : - bin_path = os.path.join(bin_path,"mac") -else: - bin_path = os.path.join(bin_path,"win") - -bin_path = os.path.abspath("test_bin") -os.environ["PATH"] += os.pathsep + bin_path - - -# bin_path = os.path.join("..","..","..","bin") -# exe = "" -# if "windows" in platform.platform().lower(): -# exe = ".exe" -# exe_path = os.path.join(bin_path, "pestpp-ies" + exe) - -# case of either appveyor, travis or local -if os.path.exists(os.path.join("pestpp","bin")): - bin_path = os.path.join("..","..","pestpp","bin") -else: - bin_path = os.path.join("..","..","..","..","pestpp","bin") +# import os +# import sys +# import shutil +# import platform +# import numpy as np +# import pandas as pd +# import platform +# import pyemu + +# bin_path = os.path.join("test_bin") +# if "linux" in platform.platform().lower(): +# bin_path = os.path.join(bin_path,"linux") +# elif "darwin" in platform.platform().lower() or "macos" in platform.platform().lower() : +# bin_path = os.path.join(bin_path,"mac") +# else: +# bin_path = os.path.join(bin_path,"win") + +# bin_path = os.path.abspath("test_bin") +# os.environ["PATH"] += os.pathsep + bin_path + + +# # bin_path = os.path.join("..","..","..","bin") +# # exe = "" +# # if "windows" in platform.platform().lower(): +# # exe = ".exe" +# # exe_path = os.path.join(bin_path, "pestpp-ies" + exe) + +# # case of either appveyor, travis or local +# if os.path.exists(os.path.join("pestpp","bin")): +# bin_path = os.path.join("..","..","pestpp","bin") +# else: +# bin_path = os.path.join("..","..","..","..","pestpp","bin") -if "windows" in platform.platform().lower(): - exe_path = os.path.join(bin_path, "win", "pestpp-ies.exe") -elif "darwin" in platform.platform().lower() or "macos" in platform.platform().lower() : - exe_path = os.path.join(bin_path, "mac", "pestpp-ies") -else: - exe_path = os.path.join(bin_path, "linux", "pestpp-ies") - -noptmax = 4 -num_reals = 20 -port = 4021 - -def basic_test(model_d="ies_10par_xsec"): - pyemu.Ensemble.reseed() - base_d = os.path.join(model_d, "template") - new_d = os.path.join(model_d, "test_template") - if os.path.exists(new_d): - shutil.rmtree(new_d) - shutil.copytree(base_d, new_d) - print(platform.platform().lower()) - pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) - print(pst.model_command) +# if "windows" in platform.platform().lower(): +# exe_path = os.path.join(bin_path, "win", "pestpp-ies.exe") +# elif "darwin" in platform.platform().lower() or "macos" in platform.platform().lower() : +# exe_path = os.path.join(bin_path, "mac", "pestpp-ies") +# else: +# exe_path = os.path.join(bin_path, "linux", "pestpp-ies") + +# noptmax = 4 +# num_reals = 20 +# port = 4021 + +# def basic_test(model_d="ies_10par_xsec"): +# pyemu.Ensemble.reseed() +# base_d = os.path.join(model_d, "template") +# new_d = os.path.join(model_d, "test_template") +# if os.path.exists(new_d): +# shutil.rmtree(new_d) +# shutil.copytree(base_d, new_d) +# print(platform.platform().lower()) +# pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) +# print(pst.model_command) - # set first par as fixed - #pst.parameter_data.loc[pst.par_names[0], "partrans"] = "fixed" - - pst.observation_data.loc[pst.nnz_obs_names,"weight"] = 1.0 - - # set noptmax - pst.control_data.noptmax = noptmax - - # wipe all pestpp options - pst.pestpp_options = {} - pst.pestpp_options["ies_num_reals"] = num_reals - pst.pestpp_options["lambda_scale_fac"] = [0.5,0.75,1.0] - pst.pestpp_options["ies_lambda_mults"] = 1.0 - # write a generic 2D cov - if os.path.exists(os.path.join(new_d,"prior.jcb")): - cov = pyemu.Cov.from_binary(os.path.join(new_d,"prior.jcb")) - #cov.to_ascii(os.path.join(new_d,"prior.cov")) - elif os.path.exists(os.path.join(new_d, "prior.cov")): - cov = pyemu.Cov.from_ascii(os.path.join(new_d, "prior.cov")) - else: - cov = pyemu.Cov.from_parameter_data(pst) - cov = pyemu.Cov(cov.as_2d, names=cov.row_names) - #cov.to_ascii(os.path.join(new_d, "prior.cov")) - cov.to_binary(os.path.join(new_d, "prior.jcb")) - - # draw some ensembles - idx = [i for i in range(num_reals)] - idx[-1] = "base" - pe = pyemu.ParameterEnsemble.from_gaussian_draw(pst, cov=cov, num_reals=num_reals) - pe.index = idx - pe.to_csv(os.path.join(new_d, "par.csv")) - pe.to_binary(os.path.join(new_d, "par.jcb")) - pe.to_csv(os.path.join(new_d, "sweep_in.csv")) - pe.loc[:, pst.adj_par_names].to_csv(os.path.join(new_d, "par_some.csv")) - pe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_par.csv")) - oe = pyemu.ObservationEnsemble.from_gaussian_draw(pst, num_reals=num_reals) - oe.index = idx - oe.to_csv(os.path.join(new_d, "obs.csv")) - oe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_base_obs.csv")) - oe.to_binary(os.path.join(new_d, "obs.jcb")) - - pst.control_data.noptmax = 0 - pst.write(os.path.join(new_d, "pest.pst")) - pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d) - df = pd.read_csv(os.path.join(new_d,"pest.phi.group.csv")) - assert df.loc[0,"head"] == 0.5,df - #return - pst.control_data.noptmax = noptmax - pst.write(os.path.join(new_d, "pest.pst")) +# # set first par as fixed +# #pst.parameter_data.loc[pst.par_names[0], "partrans"] = "fixed" + +# pst.observation_data.loc[pst.nnz_obs_names,"weight"] = 1.0 + +# # set noptmax +# pst.control_data.noptmax = noptmax + +# # wipe all pestpp options +# pst.pestpp_options = {} +# pst.pestpp_options["ies_num_reals"] = num_reals +# pst.pestpp_options["lambda_scale_fac"] = [0.5,0.75,1.0] +# pst.pestpp_options["ies_lambda_mults"] = 1.0 +# # write a generic 2D cov +# if os.path.exists(os.path.join(new_d,"prior.jcb")): +# cov = pyemu.Cov.from_binary(os.path.join(new_d,"prior.jcb")) +# #cov.to_ascii(os.path.join(new_d,"prior.cov")) +# elif os.path.exists(os.path.join(new_d, "prior.cov")): +# cov = pyemu.Cov.from_ascii(os.path.join(new_d, "prior.cov")) +# else: +# cov = pyemu.Cov.from_parameter_data(pst) +# cov = pyemu.Cov(cov.as_2d, names=cov.row_names) +# #cov.to_ascii(os.path.join(new_d, "prior.cov")) +# cov.to_binary(os.path.join(new_d, "prior.jcb")) + +# # draw some ensembles +# idx = [i for i in range(num_reals)] +# idx[-1] = "base" +# pe = pyemu.ParameterEnsemble.from_gaussian_draw(pst, cov=cov, num_reals=num_reals) +# pe.index = idx +# pe.to_csv(os.path.join(new_d, "par.csv")) +# pe.to_binary(os.path.join(new_d, "par.jcb")) +# pe.to_csv(os.path.join(new_d, "sweep_in.csv")) +# pe.loc[:, pst.adj_par_names].to_csv(os.path.join(new_d, "par_some.csv")) +# pe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_par.csv")) +# oe = pyemu.ObservationEnsemble.from_gaussian_draw(pst, num_reals=num_reals) +# oe.index = idx +# oe.to_csv(os.path.join(new_d, "obs.csv")) +# oe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_base_obs.csv")) +# oe.to_binary(os.path.join(new_d, "obs.jcb")) + +# pst.control_data.noptmax = 0 +# pst.write(os.path.join(new_d, "pest.pst")) +# pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d) +# df = pd.read_csv(os.path.join(new_d,"pest.phi.group.csv")) +# assert df.loc[0,"head"] == 0.5,df +# #return +# pst.control_data.noptmax = noptmax +# pst.write(os.path.join(new_d, "pest.pst")) - m_d = os.path.join(model_d,"master_pestpp_sen") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-sen"), "pest.pst", 5, master_dir=m_d, - worker_root=model_d,port=port,verbose=True) - #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-sen")),cwd=new_d) - df = pd.read_csv(os.path.join(m_d, "pest.mio"),index_col=0) - - # run sweep - m_d = os.path.join(model_d,"master_sweep1") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-swp"), "pest.pst", 5, master_dir=m_d, - worker_root=model_d,port=port,verbose=True) - df = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) +# m_d = os.path.join(model_d,"master_pestpp_sen") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-sen"), "pest.pst", 5, master_dir=m_d, +# worker_root=model_d,port=port,verbose=True) +# #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-sen")),cwd=new_d) +# df = pd.read_csv(os.path.join(m_d, "pest.mio"),index_col=0) + +# # run sweep +# m_d = os.path.join(model_d,"master_sweep1") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-swp"), "pest.pst", 5, master_dir=m_d, +# worker_root=model_d,port=port,verbose=True) +# df = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) - m_d = os.path.join(model_d,"master_pestpp-glm") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-glm"), "pest.pst", 10, master_dir=m_d, - worker_root=model_d,port=port,verbose=True) - #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=new_d) - df = pd.read_csv(os.path.join(m_d, "pest.par.usum.csv"),index_col=0) - - m_d = os.path.join(model_d,"master_pestpp-ies") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pyemu.os_utils.start_workers(new_d, exe_path, "pest.pst", 10, master_dir=m_d, - worker_root=model_d,port=port,verbose=True) - - - -def glm_save_binary_test(): - model_d = "ies_10par_xsec" - - t_d = os.path.join(model_d, "template") - m_d = os.path.join(model_d, "master_save_binary") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d, "pest.pst")) - pst.pestpp_options = {"glm_num_reals":30,"save_binary":True} - pst.control_data.noptmax = 1 - pst.write(os.path.join(t_d, "pest_save_binary.pst")) - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-glm"), "pest_save_binary.pst", 10, master_dir=m_d, - worker_root=model_d, port=port) - - pe = pyemu.ParameterEnsemble.from_binary(pst=pst,filename=os.path.join(m_d,"pest_save_binary.post.paren.jcb")) - pe = pyemu.ObservationEnsemble.from_binary(pst=pst,filename=os.path.join(m_d, "pest_save_binary.post.obsen.jcb")) - -def sweep_forgive_test(): - model_d = "ies_10par_xsec" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_sweep_forgive") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) - pe = pyemu.ParameterEnsemble.from_uniform_draw(pst,num_reals=50)#.loc[:,pst.par_names[:2]] - pe.loc[:,pst.par_names[2:]] = pst.parameter_data.loc[pst.par_names[2:],"parval1"].values - pe.to_csv(os.path.join(t_d,"sweep_in.csv")) - pst.pestpp_options["sweep_forgive"] = True - pst.control_data.noptmax = -1 - pst.pestpp_options["ies_num_reals"] = 5 - pst.write(os.path.join(t_d,"pest_forgive.pst")) - - - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_forgive.pst", 10, master_dir=m_d, - worker_root=model_d,port=port) - df1 = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) - - pe = pe.loc[:,pst.par_names[:2]] - pe.to_csv(os.path.join(t_d,"sweep_in.csv")) - pst.pestpp_options["sweep_forgive"] = True - pst.write(os.path.join(t_d,"pest_forgive.pst")) - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_forgive.pst", 10, master_dir=m_d, - worker_root=model_d,port=port) - df2 = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) - diff = df1 - df2 - print(diff.max()) - assert diff.max().max() == 0.0 - - - - -def inv_regul_test(): - model_d = "ies_10par_xsec" +# m_d = os.path.join(model_d,"master_pestpp-glm") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-glm"), "pest.pst", 10, master_dir=m_d, +# worker_root=model_d,port=port,verbose=True) +# #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=new_d) +# df = pd.read_csv(os.path.join(m_d, "pest.par.usum.csv"),index_col=0) + +# m_d = os.path.join(model_d,"master_pestpp-ies") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pyemu.os_utils.start_workers(new_d, exe_path, "pest.pst", 10, master_dir=m_d, +# worker_root=model_d,port=port,verbose=True) + + + +# def glm_save_binary_test(): +# model_d = "ies_10par_xsec" + +# t_d = os.path.join(model_d, "template") +# m_d = os.path.join(model_d, "master_save_binary") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d, "pest.pst")) +# pst.pestpp_options = {"glm_num_reals":30,"save_binary":True} +# pst.control_data.noptmax = 1 +# pst.write(os.path.join(t_d, "pest_save_binary.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-glm"), "pest_save_binary.pst", 10, master_dir=m_d, +# worker_root=model_d, port=port) + +# pe = pyemu.ParameterEnsemble.from_binary(pst=pst,filename=os.path.join(m_d,"pest_save_binary.post.paren.jcb")) +# pe = pyemu.ObservationEnsemble.from_binary(pst=pst,filename=os.path.join(m_d, "pest_save_binary.post.obsen.jcb")) + +# def sweep_forgive_test(): +# model_d = "ies_10par_xsec" +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_sweep_forgive") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) +# pe = pyemu.ParameterEnsemble.from_uniform_draw(pst,num_reals=50)#.loc[:,pst.par_names[:2]] +# pe.loc[:,pst.par_names[2:]] = pst.parameter_data.loc[pst.par_names[2:],"parval1"].values +# pe.to_csv(os.path.join(t_d,"sweep_in.csv")) +# pst.pestpp_options["sweep_forgive"] = True +# pst.control_data.noptmax = -1 +# pst.pestpp_options["ies_num_reals"] = 5 +# pst.write(os.path.join(t_d,"pest_forgive.pst")) + + +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_forgive.pst", 10, master_dir=m_d, +# worker_root=model_d,port=port) +# df1 = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) + +# pe = pe.loc[:,pst.par_names[:2]] +# pe.to_csv(os.path.join(t_d,"sweep_in.csv")) +# pst.pestpp_options["sweep_forgive"] = True +# pst.write(os.path.join(t_d,"pest_forgive.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_forgive.pst", 10, master_dir=m_d, +# worker_root=model_d,port=port) +# df2 = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) +# diff = df1 - df2 +# print(diff.max()) +# assert diff.max().max() == 0.0 + + + + +# def inv_regul_test(): +# model_d = "ies_10par_xsec" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_inv_regul") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) - #pyemu.helpers.zero_order_tikhonov(pst) - #pst.control_data.pestmode = "regularization" - pst.reg_data.phimlim = 2 - pst.reg_data.phimaccept = 2.2 - pst.control_data.noptmax = 10 - pst.write(os.path.join(t_d,"pest_regul.pst")) - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_regul.pst", 10, master_dir=m_d, - worker_root=model_d,port=port) +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_inv_regul") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) +# #pyemu.helpers.zero_order_tikhonov(pst) +# #pst.control_data.pestmode = "regularization" +# pst.reg_data.phimlim = 2 +# pst.reg_data.phimaccept = 2.2 +# pst.control_data.noptmax = 10 +# pst.write(os.path.join(t_d,"pest_regul.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_regul.pst", 10, master_dir=m_d, +# worker_root=model_d,port=port) -def tie_by_group_test(): - model_d = "ies_10par_xsec" +# def tie_by_group_test(): +# model_d = "ies_10par_xsec" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_tie_by_group") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) - par = pst.parameter_data - tied_names = pst.adj_par_names[:3] - par.loc[tied_names[1:3],"partrans"] = "tied" - par.loc[tied_names[1:3],"partied"] = tied_names[0] - par.loc[tied_names[1:3],"parval1"] = 1.0 - par.loc[tied_names[1:3],"parubnd"] = par.loc[tied_names[1:3],"parval1"] * 1.0001 - par.loc[tied_names[1:3],"parlbnd"] = par.loc[tied_names[1:3],"parval1"] * 0.9999 - pst.pestpp_options = {} - pst.pestpp_options["ies_num_reals"] = 10 - pst.pestpp_options["ies_lambda_mults"] = 1.0 - pst.pestpp_options["lambda_scale_fac"] = 1.0 - pst.pestpp_options["tie_by_group"] = True - pst.pestpp_options["ies_include_base"] = False - pst.pestpp_options["enforce_tied_bounds"] = True - pst.control_data.noptmax = 1 - - - pst.write(os.path.join(t_d,"pest_tied.pst")) - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "pest_tied.pst", 5, master_dir=m_d, - worker_root=model_d,port=port) - df = pd.read_csv(os.path.join(m_d,"pest_tied.sen.par.csv"),index_col=0) - df.columns = df.columns.str.lower() - print(df.loc[:,tied_names[1:3]]) - print(df.loc[:,tied_names[1:3]].std(axis=1)) - print(df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max()) - assert df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max() < 1.0e-8 - for real in df.index: - too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] - assert too_low.shape[0] == 0, "sen,{0},{1}".format(real,too_low) - too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] - assert too_high.shape[0] == 0, "sen,{0},{1}".format(real,too_high) +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_tie_by_group") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) +# par = pst.parameter_data +# tied_names = pst.adj_par_names[:3] +# par.loc[tied_names[1:3],"partrans"] = "tied" +# par.loc[tied_names[1:3],"partied"] = tied_names[0] +# par.loc[tied_names[1:3],"parval1"] = 1.0 +# par.loc[tied_names[1:3],"parubnd"] = par.loc[tied_names[1:3],"parval1"] * 1.0001 +# par.loc[tied_names[1:3],"parlbnd"] = par.loc[tied_names[1:3],"parval1"] * 0.9999 +# pst.pestpp_options = {} +# pst.pestpp_options["ies_num_reals"] = 10 +# pst.pestpp_options["ies_lambda_mults"] = 1.0 +# pst.pestpp_options["lambda_scale_fac"] = 1.0 +# pst.pestpp_options["tie_by_group"] = True +# pst.pestpp_options["ies_include_base"] = False +# pst.pestpp_options["enforce_tied_bounds"] = True +# pst.control_data.noptmax = 1 + + +# pst.write(os.path.join(t_d,"pest_tied.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "pest_tied.pst", 5, master_dir=m_d, +# worker_root=model_d,port=port) +# df = pd.read_csv(os.path.join(m_d,"pest_tied.sen.par.csv"),index_col=0) +# df.columns = df.columns.str.lower() +# print(df.loc[:,tied_names[1:3]]) +# print(df.loc[:,tied_names[1:3]].std(axis=1)) +# print(df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max()) +# assert df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max() < 1.0e-8 +# for real in df.index: +# too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] +# assert too_low.shape[0] == 0, "sen,{0},{1}".format(real,too_low) +# too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] +# assert too_high.shape[0] == 0, "sen,{0},{1}".format(real,too_high) - #pst.write(os.path.join(t_d,"pest_tied.pst")) - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_tied.pst", 5, master_dir=m_d, - worker_root=model_d,port=port) - jco = pyemu.Jco.from_binary(os.path.join(m_d,"pest_tied.jcb")) - assert jco.shape[1] == 2,jco.shape - par_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_tied.par")) - print(par_df) - too_low = par.loc[par_df.parval1 < par.parlbnd,"parnme"] - assert too_low.shape[0] == 0,too_low - too_high = par.loc[par_df.parval1 > par.parubnd, "parnme"] - assert too_high.shape[0] == 0, too_high +# #pst.write(os.path.join(t_d,"pest_tied.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_tied.pst", 5, master_dir=m_d, +# worker_root=model_d,port=port) +# jco = pyemu.Jco.from_binary(os.path.join(m_d,"pest_tied.jcb")) +# assert jco.shape[1] == 2,jco.shape +# par_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_tied.par")) +# print(par_df) +# too_low = par.loc[par_df.parval1 < par.parlbnd,"parnme"] +# assert too_low.shape[0] == 0,too_low +# too_high = par.loc[par_df.parval1 > par.parubnd, "parnme"] +# assert too_high.shape[0] == 0, too_high - pst.control_data.noptmax = 1 - pst.write(os.path.join(t_d, "pest_tied.pst")) +# pst.control_data.noptmax = 1 +# pst.write(os.path.join(t_d, "pest_tied.pst")) - pyemu.os_utils.start_workers(t_d, exe_path, "pest_tied.pst", 10, master_dir=m_d, - worker_root=model_d,port=port) +# pyemu.os_utils.start_workers(t_d, exe_path, "pest_tied.pst", 10, master_dir=m_d, +# worker_root=model_d,port=port) - for nopt in range(pst.control_data.noptmax+1): - df = pd.read_csv(os.path.join(m_d,"pest_tied.{0}.par.csv".format(nopt)),index_col=0) - df.columns = df.columns.str.lower() - print(df.loc[:,tied_names[1:3]]) - print(df.loc[:,tied_names[1:3]].std(axis=1)) - print(df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max()) - assert df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max() < 1.0e-8 - for real in df.index: - too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] - assert too_low.shape[0] == 0, "ies,{0},{1},{2}".format(nopt,real,too_low) - too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] - assert too_high.shape[0] == 0, "ies,{0},{1},{2}".format(nopt,real,too_high) - - - par.loc[tied_names[1:3],"parval1"] = par.loc[tied_names[0],"parval1"] - print(par.parval1) - par.loc[tied_names[1:3], "parubnd"] = par.loc[tied_names[1:3], "parval1"] * 1.5 - par.loc[tied_names[1:3], "parlbnd"] = par.loc[tied_names[1:3], "parval1"] * 0.5 - pst.pestpp_options["ies_num_reals"] = 10 - pst.pestpp_options["ies_lambda_mults"] = 1.0 - pst.pestpp_options["lambda_scale_fac"] = 1.0 - pst.pestpp_options["tie_by_group"] = True - - - pst.control_data.noptmax = 3 - pst.write(os.path.join(t_d, "pest_tied.pst")) - - pyemu.os_utils.start_workers(t_d, exe_path, "pest_tied.pst", 10, master_dir=m_d, - worker_root=model_d, port=port) - df = pd.read_csv(os.path.join(m_d, "pest_tied.{0}.par.csv".format(pst.control_data.noptmax)), index_col=0) - df.columns = df.columns.str.lower() - print(df.loc[:, tied_names].std(axis=1).apply(np.abs).max()) - assert df.loc[:, tied_names].std(axis=1).apply(np.abs).max() < 1.0e-8 - for real in df.index: - too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] - assert too_low.shape[0] == 0, "ies,{0},{1}".format(real,too_low) - too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] - assert too_high.shape[0] == 0, "ies,{0},{1}".format(real,too_high) +# for nopt in range(pst.control_data.noptmax+1): +# df = pd.read_csv(os.path.join(m_d,"pest_tied.{0}.par.csv".format(nopt)),index_col=0) +# df.columns = df.columns.str.lower() +# print(df.loc[:,tied_names[1:3]]) +# print(df.loc[:,tied_names[1:3]].std(axis=1)) +# print(df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max()) +# assert df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max() < 1.0e-8 +# for real in df.index: +# too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] +# assert too_low.shape[0] == 0, "ies,{0},{1},{2}".format(nopt,real,too_low) +# too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] +# assert too_high.shape[0] == 0, "ies,{0},{1},{2}".format(nopt,real,too_high) + + +# par.loc[tied_names[1:3],"parval1"] = par.loc[tied_names[0],"parval1"] +# print(par.parval1) +# par.loc[tied_names[1:3], "parubnd"] = par.loc[tied_names[1:3], "parval1"] * 1.5 +# par.loc[tied_names[1:3], "parlbnd"] = par.loc[tied_names[1:3], "parval1"] * 0.5 +# pst.pestpp_options["ies_num_reals"] = 10 +# pst.pestpp_options["ies_lambda_mults"] = 1.0 +# pst.pestpp_options["lambda_scale_fac"] = 1.0 +# pst.pestpp_options["tie_by_group"] = True + + +# pst.control_data.noptmax = 3 +# pst.write(os.path.join(t_d, "pest_tied.pst")) + +# pyemu.os_utils.start_workers(t_d, exe_path, "pest_tied.pst", 10, master_dir=m_d, +# worker_root=model_d, port=port) +# df = pd.read_csv(os.path.join(m_d, "pest_tied.{0}.par.csv".format(pst.control_data.noptmax)), index_col=0) +# df.columns = df.columns.str.lower() +# print(df.loc[:, tied_names].std(axis=1).apply(np.abs).max()) +# assert df.loc[:, tied_names].std(axis=1).apply(np.abs).max() < 1.0e-8 +# for real in df.index: +# too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] +# assert too_low.shape[0] == 0, "ies,{0},{1}".format(real,too_low) +# too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] +# assert too_high.shape[0] == 0, "ies,{0},{1}".format(real,too_high) - df.to_csv(os.path.join(t_d,"sweep_in.csv")) - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_tied.pst", 5, master_dir=m_d, - worker_root=model_d,port=port) - pst.control_data.noptmax = 3 - pst.pestpp_options["enforce_tied_bounds"] = False - pst.write(os.path.join(t_d, "pest_tied.pst")) - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_tied.pst", 5, master_dir=m_d, - worker_root=model_d,port=port) - jco = pyemu.Jco.from_binary(os.path.join(m_d,"pest_tied.jcb")) - assert jco.shape[1] == 2,jco.shape +# df.to_csv(os.path.join(t_d,"sweep_in.csv")) +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_tied.pst", 5, master_dir=m_d, +# worker_root=model_d,port=port) +# pst.control_data.noptmax = 3 +# pst.pestpp_options["enforce_tied_bounds"] = False +# pst.write(os.path.join(t_d, "pest_tied.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_tied.pst", 5, master_dir=m_d, +# worker_root=model_d,port=port) +# jco = pyemu.Jco.from_binary(os.path.join(m_d,"pest_tied.jcb")) +# assert jco.shape[1] == 2,jco.shape -def unc_file_test(): - model_d = "ies_10par_xsec" +# def unc_file_test(): +# model_d = "ies_10par_xsec" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_uncfile") - if os.path.exists(m_d): - shutil.rmtree(m_d) - shutil.copytree(t_d,m_d) - pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) - cov = pyemu.Cov.from_parameter_data(pst) - cov.to_uncfile(os.path.join(m_d,"pest.unc"),covmat_file=os.path.join(m_d,"cov.mat"),var_mult=2.0,include_path=False) - pst.pestpp_options = {} - pst.pestpp_options["parcov"] = "pest.unc" - pst.pestpp_options["ies_num_reals"] = 10000 - pst.pestpp_options["ies_verbose_level"] = 3 - pst.control_data.noptmax = -2 - pst.write(os.path.join(m_d,"pest_unc.pst")) - pyemu.os_utils.run("{0} {1}".format(exe_path,"pest_unc.pst"),cwd=m_d) - - pe_1 = pd.read_csv(os.path.join(m_d,"pest_unc.0.par.csv"),index_col=0).apply(np.log10) - - cov = pyemu.Cov.from_parameter_data(pst) - cov *= 2.0 - cov.to_uncfile(os.path.join(m_d,"pest.unc"),covmat_file=os.path.join(m_d,"cov.mat"),var_mult=1.0,include_path=False) - pst.pestpp_options = {} - pst.pestpp_options["parcov"] = "cov.mat" - pst.pestpp_options["ies_num_reals"] = 10000 - pst.pestpp_options["ies_verbose_level"] = 3 - pst.control_data.noptmax = -2 - pst.write(os.path.join(m_d,"pest_unc.pst")) - pyemu.os_utils.run("{0} {1}".format(exe_path,"pest_unc.pst"),cwd=m_d) - pe_2 = pd.read_csv(os.path.join(m_d,"pest_unc.0.par.csv"),index_col=0).apply(np.log10) - diff = pe_1 - pe_2 - print(pe_1.std(ddof=0)**2) - print(pe_2.std(ddof=0)**2) - print(diff.sum()) - assert diff.sum().max() < 1.0e-10 - - cov.to_uncfile(os.path.join(m_d, "pest.unc"), covmat_file=None) - pst.control_data.noptmax = -2 - pst.pestpp_options["ies_num_reals"] = 100000 - pst.pestpp_options["ies_enforce_bounds"] = False - pst.write(os.path.join(m_d, "pest_unc.pst")) - pyemu.os_utils.run("{0} {1}".format(exe_path, "pest_unc.pst"), cwd=m_d) - pe_3 = pd.read_csv(os.path.join(m_d, "pest_unc.0.par.csv"), index_col=0).apply(np.log10) - print(pe_3.std(ddof=0)) - pe_std = pe_3.std(ddof=0) - for r,v in zip(cov.row_names,cov.x): - d = np.abs(pe_std.loc[r] - np.sqrt(v)) - - print(r,v,np.sqrt(v),d) - assert d < 0.01 - pst.control_data.noptmax = -1 - pst.write(os.path.join(m_d, "pest_unc.pst")) - pyemu.os_utils.run("{0} {1}".format(exe_path.replace("-ies","-glm"), "pest_unc.pst"), cwd=m_d) - fosm_df = pd.read_csv(os.path.join(m_d,"pest_unc.par.usum.csv"),index_col=0) - cov_df = cov.to_dataframe() - for pname,prior_std in zip(fosm_df.index,fosm_df.prior_stdev): - d = np.abs(prior_std - np.sqrt(cov_df.loc[pname,pname])) - print(pname,d) - assert d < 1.0e-4 - - -def parchglim_test(): - model_d = "ies_10par_xsec" +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_uncfile") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# shutil.copytree(t_d,m_d) +# pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) +# cov = pyemu.Cov.from_parameter_data(pst) +# cov.to_uncfile(os.path.join(m_d,"pest.unc"),covmat_file=os.path.join(m_d,"cov.mat"),var_mult=2.0,include_path=False) +# pst.pestpp_options = {} +# pst.pestpp_options["parcov"] = "pest.unc" +# pst.pestpp_options["ies_num_reals"] = 10000 +# pst.pestpp_options["ies_verbose_level"] = 3 +# pst.control_data.noptmax = -2 +# pst.write(os.path.join(m_d,"pest_unc.pst")) +# pyemu.os_utils.run("{0} {1}".format(exe_path,"pest_unc.pst"),cwd=m_d) + +# pe_1 = pd.read_csv(os.path.join(m_d,"pest_unc.0.par.csv"),index_col=0).apply(np.log10) + +# cov = pyemu.Cov.from_parameter_data(pst) +# cov *= 2.0 +# cov.to_uncfile(os.path.join(m_d,"pest.unc"),covmat_file=os.path.join(m_d,"cov.mat"),var_mult=1.0,include_path=False) +# pst.pestpp_options = {} +# pst.pestpp_options["parcov"] = "cov.mat" +# pst.pestpp_options["ies_num_reals"] = 10000 +# pst.pestpp_options["ies_verbose_level"] = 3 +# pst.control_data.noptmax = -2 +# pst.write(os.path.join(m_d,"pest_unc.pst")) +# pyemu.os_utils.run("{0} {1}".format(exe_path,"pest_unc.pst"),cwd=m_d) +# pe_2 = pd.read_csv(os.path.join(m_d,"pest_unc.0.par.csv"),index_col=0).apply(np.log10) +# diff = pe_1 - pe_2 +# print(pe_1.std(ddof=0)**2) +# print(pe_2.std(ddof=0)**2) +# print(diff.sum()) +# assert diff.sum().max() < 1.0e-10 + +# cov.to_uncfile(os.path.join(m_d, "pest.unc"), covmat_file=None) +# pst.control_data.noptmax = -2 +# pst.pestpp_options["ies_num_reals"] = 100000 +# pst.pestpp_options["ies_enforce_bounds"] = False +# pst.write(os.path.join(m_d, "pest_unc.pst")) +# pyemu.os_utils.run("{0} {1}".format(exe_path, "pest_unc.pst"), cwd=m_d) +# pe_3 = pd.read_csv(os.path.join(m_d, "pest_unc.0.par.csv"), index_col=0).apply(np.log10) +# print(pe_3.std(ddof=0)) +# pe_std = pe_3.std(ddof=0) +# for r,v in zip(cov.row_names,cov.x): +# d = np.abs(pe_std.loc[r] - np.sqrt(v)) + +# print(r,v,np.sqrt(v),d) +# assert d < 0.01 +# pst.control_data.noptmax = -1 +# pst.write(os.path.join(m_d, "pest_unc.pst")) +# pyemu.os_utils.run("{0} {1}".format(exe_path.replace("-ies","-glm"), "pest_unc.pst"), cwd=m_d) +# fosm_df = pd.read_csv(os.path.join(m_d,"pest_unc.par.usum.csv"),index_col=0) +# cov_df = cov.to_dataframe() +# for pname,prior_std in zip(fosm_df.index,fosm_df.prior_stdev): +# d = np.abs(prior_std - np.sqrt(cov_df.loc[pname,pname])) +# print(pname,d) +# assert d < 1.0e-4 + + +# def parchglim_test(): +# model_d = "ies_10par_xsec" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_parchglim") - if os.path.exists(m_d): - shutil.rmtree(m_d) - shutil.copytree(t_d,m_d) - pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) - fpm = 1.05 - pst.control_data.facparmax = fpm - par = pst.parameter_data - par.loc[pst.par_names[1:],"partrans"] = "fixed" - par.loc[pst.par_names[0],"partrans"] = "log" - par.loc[pst.par_names[0],"parchglim"] = "factor" - par.loc[pst.par_names[0],"parval1"] = 1.0 +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_parchglim") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# shutil.copytree(t_d,m_d) +# pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) +# fpm = 1.05 +# pst.control_data.facparmax = fpm +# par = pst.parameter_data +# par.loc[pst.par_names[1:],"partrans"] = "fixed" +# par.loc[pst.par_names[0],"partrans"] = "log" +# par.loc[pst.par_names[0],"parchglim"] = "factor" +# par.loc[pst.par_names[0],"parval1"] = 1.0 - pst.control_data.noptmax = 1 - pst.pestpp_options["lambdas"] = 1.0 - pst.pestpp_options["lambda_scale_fac"] = 1.0 - pst.write(os.path.join(m_d,"pest_parchglim.pst")) - pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) - p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) - print(p_df.loc["stage","parval1"],fpm) - assert p_df.loc["stage","parval1"] == fpm - - rpm = 0.1 - par.loc[pst.par_names[0],"parchglim"] = "relative" - pst.control_data.relparmax = rpm - pst.write(os.path.join(m_d,"pest_parchglim.pst")) - pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) - p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) - print(par) - print(p_df) - assert p_df.loc["stage","parval1"] == par.loc["stage","parval1"] + (rpm * par.loc["stage","parval1"]) - - - par.loc[pst.par_names[0],"partrans"] = "none" - par.loc[pst.par_names[0],"parlbnd"] = -10.0 - par.loc[pst.par_names[0],"parubnd"] = 0.0 - par.loc[pst.par_names[0],"parchglim"] = "factor" - par.loc[pst.par_names[0],"parval1"] = -1.0 - pst.write(os.path.join(m_d,"pest_parchglim.pst")) - pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) - p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) - print(p_df) - print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + np.abs(par.loc["stage","parval1"] * (fpm-1))) - assert p_df.loc["stage","parval1"] <= par.loc["stage","parval1"] + np.abs(par.loc["stage","parval1"] * (fpm-1)) - - rpm = 1.1 - par.loc[pst.par_names[0],"partrans"] = "none" - par.loc[pst.par_names[0],"parlbnd"] = -10.0 - par.loc[pst.par_names[0],"parubnd"] = 10.0 - par.loc[pst.par_names[0],"parchglim"] = "relative" - par.loc[pst.par_names[0],"parval1"] = -1.0 - pst.control_data.relparmax = rpm - pst.write(os.path.join(m_d,"pest_parchglim.pst")) - pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) - p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) - print(p_df) - print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + rpm) - assert np.abs(p_df.loc["stage","parval1"] - (par.loc["stage","parval1"] + rpm)) < 1.0e-6 - - - par.loc[pst.par_names[1:],"partrans"] = "log" - par.loc[pst.par_names[1:],"parchglim"] = "factor" - pst.control_data.facparmax = 5.0 +# pst.control_data.noptmax = 1 +# pst.pestpp_options["lambdas"] = 1.0 +# pst.pestpp_options["lambda_scale_fac"] = 1.0 +# pst.write(os.path.join(m_d,"pest_parchglim.pst")) +# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) +# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) +# print(p_df.loc["stage","parval1"],fpm) +# assert p_df.loc["stage","parval1"] == fpm + +# rpm = 0.1 +# par.loc[pst.par_names[0],"parchglim"] = "relative" +# pst.control_data.relparmax = rpm +# pst.write(os.path.join(m_d,"pest_parchglim.pst")) +# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) +# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) +# print(par) +# print(p_df) +# assert p_df.loc["stage","parval1"] == par.loc["stage","parval1"] + (rpm * par.loc["stage","parval1"]) + + +# par.loc[pst.par_names[0],"partrans"] = "none" +# par.loc[pst.par_names[0],"parlbnd"] = -10.0 +# par.loc[pst.par_names[0],"parubnd"] = 0.0 +# par.loc[pst.par_names[0],"parchglim"] = "factor" +# par.loc[pst.par_names[0],"parval1"] = -1.0 +# pst.write(os.path.join(m_d,"pest_parchglim.pst")) +# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) +# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) +# print(p_df) +# print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + np.abs(par.loc["stage","parval1"] * (fpm-1))) +# assert p_df.loc["stage","parval1"] <= par.loc["stage","parval1"] + np.abs(par.loc["stage","parval1"] * (fpm-1)) + +# rpm = 1.1 +# par.loc[pst.par_names[0],"partrans"] = "none" +# par.loc[pst.par_names[0],"parlbnd"] = -10.0 +# par.loc[pst.par_names[0],"parubnd"] = 10.0 +# par.loc[pst.par_names[0],"parchglim"] = "relative" +# par.loc[pst.par_names[0],"parval1"] = -1.0 +# pst.control_data.relparmax = rpm +# pst.write(os.path.join(m_d,"pest_parchglim.pst")) +# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) +# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) +# print(p_df) +# print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + rpm) +# assert np.abs(p_df.loc["stage","parval1"] - (par.loc["stage","parval1"] + rpm)) < 1.0e-6 + + +# par.loc[pst.par_names[1:],"partrans"] = "log" +# par.loc[pst.par_names[1:],"parchglim"] = "factor" +# pst.control_data.facparmax = 5.0 - pst.write(os.path.join(m_d,"pest_parchglim.pst")) - pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) - p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) - print(p_df) - print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + rpm) - d = np.abs(p_df.loc["stage","parval1"] - (par.loc["stage","parval1"] + rpm)) - assert d < 1.0e-6,d - - rpm = 1.1 - par.loc[pst.par_names[1:],"partrans"] = "fixed" - par.loc[pst.par_names[1:],"parchglim"] = "factor" - par.loc[pst.par_names[0],"partrans"] = "none" - par.loc[pst.par_names[0],"parlbnd"] = -10.0 - par.loc[pst.par_names[0],"parubnd"] = 10.0 - par.loc[pst.par_names[0],"parchglim"] = "relative" - par.loc[pst.par_names[0],"parval1"] = 0.0 - pst.control_data.relparmax = rpm - pst.write(os.path.join(m_d,"pest_parchglim.pst")) - try: - - pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) - except: - pass - else: - raise Exception("should have failed") +# pst.write(os.path.join(m_d,"pest_parchglim.pst")) +# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) +# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) +# print(p_df) +# print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + rpm) +# d = np.abs(p_df.loc["stage","parval1"] - (par.loc["stage","parval1"] + rpm)) +# assert d < 1.0e-6,d + +# rpm = 1.1 +# par.loc[pst.par_names[1:],"partrans"] = "fixed" +# par.loc[pst.par_names[1:],"parchglim"] = "factor" +# par.loc[pst.par_names[0],"partrans"] = "none" +# par.loc[pst.par_names[0],"parlbnd"] = -10.0 +# par.loc[pst.par_names[0],"parubnd"] = 10.0 +# par.loc[pst.par_names[0],"parchglim"] = "relative" +# par.loc[pst.par_names[0],"parval1"] = 0.0 +# pst.control_data.relparmax = rpm +# pst.write(os.path.join(m_d,"pest_parchglim.pst")) +# try: + +# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) +# except: +# pass +# else: +# raise Exception("should have failed") - rpm = 100 - fpm = 100 - par.loc[pst.par_names[1:],"partrans"] = "fixed" - par.loc[pst.par_names[1:],"parchglim"] = "factor" - par.loc[pst.par_names[0],"partrans"] = "none" - par.loc[pst.par_names[0],"parlbnd"] = 0.9 - par.loc[pst.par_names[0],"parubnd"] = 1.1 - par.loc[pst.par_names[0],"parchglim"] = "relative" - par.loc[pst.par_names[0],"parval1"] = 1.0 - pst.control_data.relparmax = rpm - pst.control_data.facparmax = fpm +# rpm = 100 +# fpm = 100 +# par.loc[pst.par_names[1:],"partrans"] = "fixed" +# par.loc[pst.par_names[1:],"parchglim"] = "factor" +# par.loc[pst.par_names[0],"partrans"] = "none" +# par.loc[pst.par_names[0],"parlbnd"] = 0.9 +# par.loc[pst.par_names[0],"parubnd"] = 1.1 +# par.loc[pst.par_names[0],"parchglim"] = "relative" +# par.loc[pst.par_names[0],"parval1"] = 1.0 +# pst.control_data.relparmax = rpm +# pst.control_data.facparmax = fpm - pst.write(os.path.join(m_d,"pest_parchglim.pst")) - pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) - p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) - print(p_df) - assert p_df.loc["stage","parval1"] == par.loc["stage","parubnd"] +# pst.write(os.path.join(m_d,"pest_parchglim.pst")) +# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) +# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) +# print(p_df) +# assert p_df.loc["stage","parval1"] == par.loc["stage","parubnd"] -def sen_plusplus_test(): - model_d = "ies_10par_xsec" +# def sen_plusplus_test(): +# model_d = "ies_10par_xsec" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_sen_plusplus") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) - pst.pestpp_options = {} - pst.pestpp_options["gsa_method"] = "morris" - pst.pestpp_options["gsa_sobol_samples"] = 50 - pst.pestpp_options["gsa_sobol_par_dist"] = "unif" - pst.pestpp_options["gsa_morris_r"] = 4 - pst.pestpp_options["gsa_morris_p"] = 5 - pst.pestpp_options["gsa_morris_delta"] = 2 - pst.write(os.path.join(t_d,"pest_sen.pst")) - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "pest_sen.pst", 5, master_dir=m_d, - worker_root=model_d,port=port) - -def secondary_marker_test(): - t_d = os.path.join("secondary_marker_test","template") - tpl_file = os.path.join(t_d,"par.dat.tpl") - - with open(tpl_file,'w') as f: - f.write("ptf ~\n") - f.write("~ p1 ~\n") - - tpl_file = "par.dat.tpl" - b_d = os.getcwd() - os.chdir(t_d) - try: - - ins_files = [f for f in os.listdir(".") if f.endswith(".ins")] - with open("forward_run.py",'w') as f: - f.write("import shutil\n") - for ins_file in ins_files: - out_file = ins_file.replace(".ins","") - f.write("shutil.copy2('{0}','{1}')\n".format(out_file+"_bak",out_file)) - - for ins_file in ins_files: - - shutil.copy2(out_file+"_bak",out_file) - pst = pyemu.Pst.from_io_files(tpl_file,tpl_file.replace(".tpl",""), - ins_file,ins_file.replace(".ins","")) - pst.control_data.noptmax = 0 - pst.pestpp_options["additional_ins_delimiters"] = "|" - pst.model_command = "python forward_run.py" - pst.write(os.path.join("test.pst")) - - pyemu.os_utils.run("{0} test.pst".format(exe_path)) - pst = pyemu.Pst("test.pst") - assert pst.res is not None - d = pst.res.loc[pst.obs_names,"modelled"] - pst.observation_data.loc[pst.obs_names,"obsval"] - l2_d = (d.apply(np.abs)**2).sum() +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_sen_plusplus") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) +# pst.pestpp_options = {} +# pst.pestpp_options["gsa_method"] = "morris" +# pst.pestpp_options["gsa_sobol_samples"] = 50 +# pst.pestpp_options["gsa_sobol_par_dist"] = "unif" +# pst.pestpp_options["gsa_morris_r"] = 4 +# pst.pestpp_options["gsa_morris_p"] = 5 +# pst.pestpp_options["gsa_morris_delta"] = 2 +# pst.write(os.path.join(t_d,"pest_sen.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "pest_sen.pst", 5, master_dir=m_d, +# worker_root=model_d,port=port) + +# def secondary_marker_test(): +# t_d = os.path.join("secondary_marker_test","template") +# tpl_file = os.path.join(t_d,"par.dat.tpl") + +# with open(tpl_file,'w') as f: +# f.write("ptf ~\n") +# f.write("~ p1 ~\n") + +# tpl_file = "par.dat.tpl" +# b_d = os.getcwd() +# os.chdir(t_d) +# try: + +# ins_files = [f for f in os.listdir(".") if f.endswith(".ins")] +# with open("forward_run.py",'w') as f: +# f.write("import shutil\n") +# for ins_file in ins_files: +# out_file = ins_file.replace(".ins","") +# f.write("shutil.copy2('{0}','{1}')\n".format(out_file+"_bak",out_file)) + +# for ins_file in ins_files: + +# shutil.copy2(out_file+"_bak",out_file) +# pst = pyemu.Pst.from_io_files(tpl_file,tpl_file.replace(".tpl",""), +# ins_file,ins_file.replace(".ins","")) +# pst.control_data.noptmax = 0 +# pst.pestpp_options["additional_ins_delimiters"] = "|" +# pst.model_command = "python forward_run.py" +# pst.write(os.path.join("test.pst")) + +# pyemu.os_utils.run("{0} test.pst".format(exe_path)) +# pst = pyemu.Pst("test.pst") +# assert pst.res is not None +# d = pst.res.loc[pst.obs_names,"modelled"] - pst.observation_data.loc[pst.obs_names,"obsval"] +# l2_d = (d.apply(np.abs)**2).sum() - except Exception as e: - os.chdir(b_d) - raise Exception(e) - os.chdir(b_d) - -def sen_basic_test(): - model_d = "sen_invest" - t_d = os.path.join(model_d, "template") - if os.path.exists(t_d): - shutil.rmtree(t_d) - os.makedirs(t_d) - par_names = ["p1","p2"] - obs_names = ["p1","p2","p1+p2","p1*p2","p1^p2","const"] - - tpl_file = os.path.join(t_d,"in.dat.tpl") - with open(tpl_file,'w') as f: - f.write("ptf ~\n") - for par_name in par_names: - f.write("{0} ~ {0} ~\n".format(par_name)) - ins_file = os.path.join(t_d,"out.dat.ins") - with open(ins_file,'w') as f: - f.write("pif ~\n") - for obs_name in obs_names: - f.write("l1 w !{0}!\n".format(obs_name)) - - with open(os.path.join(t_d,"forward_run.py"),'w') as f: - f.write("import pandas as pd\n") - f.write("df = pd.read_csv('in.dat',index_col=0,delim_whitespace=True,names=['name','value'])\n") - f.write("df.loc['p1+p2','value'] = df.loc['p1','value'] + df.loc['p2','value']\n") - f.write("df.loc['p1*p2','value'] = df.loc['p1','value'] * df.loc['p2','value']\n") - f.write("df.loc['p1^p2','value'] = df.loc['p1','value'] * df.loc['p2','value']\n") - f.write("df.loc['const','value'] = 1.0\n") - f.write("df.to_csv('out.dat',sep=' ',header=False)\n") - - with open(os.path.join(t_d,"in.dat"),'w') as f: - f.write("p1 1.0\n") - f.write("p2 1.0\n") - f.write("p3 1.0\n") - pyemu.os_utils.run("python forward_run.py",cwd=t_d) - - pst = pyemu.Pst.from_io_files(tpl_files=tpl_file,in_files=tpl_file.replace(".tpl",""), - ins_files=ins_file,out_files=ins_file.replace(".ins",""),pst_path=".") - pst.model_command = "python forward_run.py" - pst.control_data.noptmax = 0 - pst.parameter_data.loc[:,"partrans"] = "log" - pst.parameter_data.loc[:,"parchglim"] = "relative" - pst.parameter_data.loc[:,"parubnd"] = 10.0 - pst.parameter_data.loc[:,"parlbnd"] = .1 - pst.parameter_data.loc[:,"parval1"] = 1.0 - - msn_file = os.path.join(t_d,"pest.msn") - mio_file = os.path.join(t_d, "pest.mio") - - obs = pst.observation_data - obs.loc[:, "weight"] = 0.0 - obs.loc["const", "weight"] = 1.0 - pst.write(os.path.join(t_d, "pest.pst")) - pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies", "-sen")), cwd=t_d) - df = pd.read_csv(msn_file, index_col=0) - df.columns = df.columns.map(str.lower) - df.columns = df.columns.map(str.strip) - df.index = df.index.map(str.lower) - print(df) - assert df.sen_mean_abs.sum() == 0.0 - assert df.sen_std_dev.sum() == 0.0 - df = pd.read_csv(mio_file, index_col=0) - df.columns = df.columns.map(str.lower) - - df.loc[:, "parameter_name"] = df.parameter_name.apply(str.lower) - df.index = df.index.map(str.lower) - print(df) - assert df.loc[df.parameter_name == "p2", :].loc["p1", "sen_mean_abs"] == 0 - assert df.loc[df.parameter_name == "p1", :].loc["p2", "sen_mean_abs"] == 0 - - pst.pestpp_options["gsa_method"] = "sobol" - pst.pestpp_options["gsa_sobol_samples"] = 5 - pst.write(os.path.join(t_d, "pest.pst")) - #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies", "-sen")), cwd=t_d) - m_d = os.path.join(model_d,"master_sobol") - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-sen"), "pest.pst", 5, master_dir=m_d, - worker_root=model_d, port=port) - si_vals = pd.read_csv(os.path.join(m_d,"pest.sobol.si.csv"),index_col=0) - sti_vals = pd.read_csv(os.path.join(m_d,"pest.sobol.sti.csv"),index_col=0) - v_d = os.path.join("sen_invest","verf") - si_verf_vals = pd.read_csv(os.path.join(v_d, "si.csv"), index_col=0) - sti_verf_vals = pd.read_csv(os.path.join(v_d, "sti.csv"), index_col=0) - d_si = (si_vals.loc[pst.obs_names,:] - si_verf_vals.loc[pst.obs_names,:]).apply(np.abs) - print(d_si.max()) - assert d_si.max().max() < .001 - d_sti = (sti_vals.loc[pst.obs_names, :] - sti_verf_vals.loc[pst.obs_names, :]).apply(np.abs) - print(d_sti.max()) - assert d_sti.max().max() < .001 - - -def salib_verf(): - import pyemu - from SALib.sample import saltelli - from SALib.analyze import sobol - m_d = os.path.join("sen_invest","master_sobol") - v_d = os.path.join("sen_invest","verf") - if os.path.exists(v_d): - shutil.rmtree(v_d) - os.makedirs(v_d) - pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) - pst.add_transform_columns() - bounds = [[l,u] for l,u in zip(pst.parameter_data.parlbnd_trans,pst.parameter_data.parubnd_trans)] - problem = {"num_vars":pst.npar_adj,"names":pst.par_names,"bounds":bounds} - test = saltelli.sample(problem,100,calc_second_order=False) - out_df = pd.read_csv(os.path.join(m_d,"pest.sobol.obs.csv"),index_col=0) - reorder_df = out_df.copy() - idx = [0,3,1,2] - for i in range(4): - s = i*5 - e = s + 5 - chunk = out_df.iloc[s:e,:].copy() - reorder_df.iloc[idx[i]::4,:] = chunk.values - print(chunk.p1,reorder_df.iloc[idx[i]::4,:].p1) - pass - si_vals = pd.DataFrame(columns=pst.par_names,index=pst.obs_names) - sti_vals = pd.DataFrame(columns=pst.par_names, index=pst.obs_names) - - for obs_name in pst.obs_names: - #if obs_name != "p1": - # continue - si = sobol.analyze(problem,reorder_df.loc[:,obs_name].values,calc_second_order=False,num_resamples=5) - print(obs_name,si) - si_vals.loc[obs_name,:] = si["S1"] - sti_vals.loc[obs_name, :] = si["ST"] - si_vals.to_csv(os.path.join(v_d,"si.csv")) - sti_vals.to_csv(os.path.join(v_d, "sti.csv")) - - # in_df = pd.read_csv(os.path.join(m_d,"pest.sobol.par.csv"),index_col=0) - # import matplotlib.pyplot as plt - # fig, ax = plt.subplots(1,1) - # print(test) - # test = test ** 10 - # print(test) - # ax.scatter(test[:,0],test[:,1],marker='.',color='g') - # ax.scatter(in_df.iloc[:,0],in_df.iloc[:,1],marker='.',color='r') - # - # - # plt.show() - - -def tplins1_test(): - model_d = "tplins_test_1" - t_d = os.path.join(model_d, "test") - if os.path.exists(t_d): - shutil.rmtree(t_d) - shutil.copytree(os.path.join(model_d,"template"),t_d) - pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) - ins_file = os.path.join(t_d,"AOC_obs.txt.ins") - pst.add_observations(ins_file,ins_file.replace(".ins",""),pst_path=".") - pst.parameter_data.loc["k_10","parval1"] = 12345 - pst.parameter_data.loc["k_10","parubnd"] = 200000 - pst.pestpp_options["tpl_force_decimal"] = True - pst.control_data.noptmax = 0 - pst.write(os.path.join(t_d,"pest.pst")) - pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=t_d) - with open(os.path.join(t_d,"hk_Layer_1.ref"),'r') as f: - for line in f: - if "e" in line.lower(): - raise Exception(line) - pst.pestpp_options.pop("tpl_force_decimal") - pst.control_data.noptmax = -1 - pst.parameter_data.loc["k_10","parval1"] = 120 - pst.parameter_data.loc["k_10","parubnd"] = 200 - pst.write(os.path.join(t_d,"pest.pst")) +# except Exception as e: +# os.chdir(b_d) +# raise Exception(e) +# os.chdir(b_d) + +# def sen_basic_test(): +# model_d = "sen_invest" +# t_d = os.path.join(model_d, "template") +# if os.path.exists(t_d): +# shutil.rmtree(t_d) +# os.makedirs(t_d) +# par_names = ["p1","p2"] +# obs_names = ["p1","p2","p1+p2","p1*p2","p1^p2","const"] + +# tpl_file = os.path.join(t_d,"in.dat.tpl") +# with open(tpl_file,'w') as f: +# f.write("ptf ~\n") +# for par_name in par_names: +# f.write("{0} ~ {0} ~\n".format(par_name)) +# ins_file = os.path.join(t_d,"out.dat.ins") +# with open(ins_file,'w') as f: +# f.write("pif ~\n") +# for obs_name in obs_names: +# f.write("l1 w !{0}!\n".format(obs_name)) + +# with open(os.path.join(t_d,"forward_run.py"),'w') as f: +# f.write("import pandas as pd\n") +# f.write("df = pd.read_csv('in.dat',index_col=0,delim_whitespace=True,names=['name','value'])\n") +# f.write("df.loc['p1+p2','value'] = df.loc['p1','value'] + df.loc['p2','value']\n") +# f.write("df.loc['p1*p2','value'] = df.loc['p1','value'] * df.loc['p2','value']\n") +# f.write("df.loc['p1^p2','value'] = df.loc['p1','value'] * df.loc['p2','value']\n") +# f.write("df.loc['const','value'] = 1.0\n") +# f.write("df.to_csv('out.dat',sep=' ',header=False)\n") + +# with open(os.path.join(t_d,"in.dat"),'w') as f: +# f.write("p1 1.0\n") +# f.write("p2 1.0\n") +# f.write("p3 1.0\n") +# pyemu.os_utils.run("python forward_run.py",cwd=t_d) + +# pst = pyemu.Pst.from_io_files(tpl_files=tpl_file,in_files=tpl_file.replace(".tpl",""), +# ins_files=ins_file,out_files=ins_file.replace(".ins",""),pst_path=".") +# pst.model_command = "python forward_run.py" +# pst.control_data.noptmax = 0 +# pst.parameter_data.loc[:,"partrans"] = "log" +# pst.parameter_data.loc[:,"parchglim"] = "relative" +# pst.parameter_data.loc[:,"parubnd"] = 10.0 +# pst.parameter_data.loc[:,"parlbnd"] = .1 +# pst.parameter_data.loc[:,"parval1"] = 1.0 + +# msn_file = os.path.join(t_d,"pest.msn") +# mio_file = os.path.join(t_d, "pest.mio") + +# obs = pst.observation_data +# obs.loc[:, "weight"] = 0.0 +# obs.loc["const", "weight"] = 1.0 +# pst.write(os.path.join(t_d, "pest.pst")) +# pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies", "-sen")), cwd=t_d) +# df = pd.read_csv(msn_file, index_col=0) +# df.columns = df.columns.map(str.lower) +# df.columns = df.columns.map(str.strip) +# df.index = df.index.map(str.lower) +# print(df) +# assert df.sen_mean_abs.sum() == 0.0 +# assert df.sen_std_dev.sum() == 0.0 +# df = pd.read_csv(mio_file, index_col=0) +# df.columns = df.columns.map(str.lower) + +# df.loc[:, "parameter_name"] = df.parameter_name.apply(str.lower) +# df.index = df.index.map(str.lower) +# print(df) +# assert df.loc[df.parameter_name == "p2", :].loc["p1", "sen_mean_abs"] == 0 +# assert df.loc[df.parameter_name == "p1", :].loc["p2", "sen_mean_abs"] == 0 + +# pst.pestpp_options["gsa_method"] = "sobol" +# pst.pestpp_options["gsa_sobol_samples"] = 5 +# pst.write(os.path.join(t_d, "pest.pst")) +# #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies", "-sen")), cwd=t_d) +# m_d = os.path.join(model_d,"master_sobol") +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-sen"), "pest.pst", 5, master_dir=m_d, +# worker_root=model_d, port=port) +# si_vals = pd.read_csv(os.path.join(m_d,"pest.sobol.si.csv"),index_col=0) +# sti_vals = pd.read_csv(os.path.join(m_d,"pest.sobol.sti.csv"),index_col=0) +# v_d = os.path.join("sen_invest","verf") +# si_verf_vals = pd.read_csv(os.path.join(v_d, "si.csv"), index_col=0) +# sti_verf_vals = pd.read_csv(os.path.join(v_d, "sti.csv"), index_col=0) +# d_si = (si_vals.loc[pst.obs_names,:] - si_verf_vals.loc[pst.obs_names,:]).apply(np.abs) +# print(d_si.max()) +# assert d_si.max().max() < .001 +# d_sti = (sti_vals.loc[pst.obs_names, :] - sti_verf_vals.loc[pst.obs_names, :]).apply(np.abs) +# print(d_sti.max()) +# assert d_sti.max().max() < .001 + + +# def salib_verf(): +# import pyemu +# from SALib.sample import saltelli +# from SALib.analyze import sobol +# m_d = os.path.join("sen_invest","master_sobol") +# v_d = os.path.join("sen_invest","verf") +# if os.path.exists(v_d): +# shutil.rmtree(v_d) +# os.makedirs(v_d) +# pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) +# pst.add_transform_columns() +# bounds = [[l,u] for l,u in zip(pst.parameter_data.parlbnd_trans,pst.parameter_data.parubnd_trans)] +# problem = {"num_vars":pst.npar_adj,"names":pst.par_names,"bounds":bounds} +# test = saltelli.sample(problem,100,calc_second_order=False) +# out_df = pd.read_csv(os.path.join(m_d,"pest.sobol.obs.csv"),index_col=0) +# reorder_df = out_df.copy() +# idx = [0,3,1,2] +# for i in range(4): +# s = i*5 +# e = s + 5 +# chunk = out_df.iloc[s:e,:].copy() +# reorder_df.iloc[idx[i]::4,:] = chunk.values +# print(chunk.p1,reorder_df.iloc[idx[i]::4,:].p1) +# pass +# si_vals = pd.DataFrame(columns=pst.par_names,index=pst.obs_names) +# sti_vals = pd.DataFrame(columns=pst.par_names, index=pst.obs_names) + +# for obs_name in pst.obs_names: +# #if obs_name != "p1": +# # continue +# si = sobol.analyze(problem,reorder_df.loc[:,obs_name].values,calc_second_order=False,num_resamples=5) +# print(obs_name,si) +# si_vals.loc[obs_name,:] = si["S1"] +# sti_vals.loc[obs_name, :] = si["ST"] +# si_vals.to_csv(os.path.join(v_d,"si.csv")) +# sti_vals.to_csv(os.path.join(v_d, "sti.csv")) + +# # in_df = pd.read_csv(os.path.join(m_d,"pest.sobol.par.csv"),index_col=0) +# # import matplotlib.pyplot as plt +# # fig, ax = plt.subplots(1,1) +# # print(test) +# # test = test ** 10 +# # print(test) +# # ax.scatter(test[:,0],test[:,1],marker='.',color='g') +# # ax.scatter(in_df.iloc[:,0],in_df.iloc[:,1],marker='.',color='r') +# # +# # +# # plt.show() + + +# def tplins1_test(): +# model_d = "tplins_test_1" +# t_d = os.path.join(model_d, "test") +# if os.path.exists(t_d): +# shutil.rmtree(t_d) +# shutil.copytree(os.path.join(model_d,"template"),t_d) +# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) +# ins_file = os.path.join(t_d,"AOC_obs.txt.ins") +# pst.add_observations(ins_file,ins_file.replace(".ins",""),pst_path=".") +# pst.parameter_data.loc["k_10","parval1"] = 12345 +# pst.parameter_data.loc["k_10","parubnd"] = 200000 +# pst.pestpp_options["tpl_force_decimal"] = True +# pst.control_data.noptmax = 0 +# pst.write(os.path.join(t_d,"pest.pst")) +# pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=t_d) +# with open(os.path.join(t_d,"hk_Layer_1.ref"),'r') as f: +# for line in f: +# if "e" in line.lower(): +# raise Exception(line) +# pst.pestpp_options.pop("tpl_force_decimal") +# pst.control_data.noptmax = -1 +# pst.parameter_data.loc["k_10","parval1"] = 120 +# pst.parameter_data.loc["k_10","parubnd"] = 200 +# pst.write(os.path.join(t_d,"pest.pst")) - pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) - obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"]) - obf_df.index = obf_df.obsnme - pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) - res_df = pst.res +# pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) +# obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"]) +# obf_df.index = obf_df.obsnme +# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) +# res_df = pst.res - d = (obf_df.obsval - res_df.modelled).apply(np.abs) - #print(d) - print(d.max()) - assert d.max() < 1.0e-5, d - - jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest.jcb")).to_dataframe().apply(np.abs) - assert jco.sum().sum() == 0, jco.sum() - - pst.control_data.noptmax = 0 - pst.write(os.path.join(t_d,"pest.pst")) - pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) - - # check the input file - the last two number should be the same - arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref")) - assert arr[-2] == arr[-1],arr[-2] - arr[-1] - - lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines() - lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines() - assert len(lines_tpl) - 1 == len(lines_in) - - pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) - pst.control_data.noptmax = -1 - pst.pestpp_options["fill_tpl_zeros"] = True - pst.write(os.path.join(t_d,"pest_fill.pst")) - pyemu.os_utils.run("{0} pest_fill.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) - obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"]) - obf_df.index = obf_df.obsnme - pst = pyemu.Pst(os.path.join(t_d,"pest_fill.pst")) - res_df = pst.res +# d = (obf_df.obsval - res_df.modelled).apply(np.abs) +# #print(d) +# print(d.max()) +# assert d.max() < 1.0e-5, d + +# jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest.jcb")).to_dataframe().apply(np.abs) +# assert jco.sum().sum() == 0, jco.sum() + +# pst.control_data.noptmax = 0 +# pst.write(os.path.join(t_d,"pest.pst")) +# pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) + +# # check the input file - the last two number should be the same +# arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref")) +# assert arr[-2] == arr[-1],arr[-2] - arr[-1] + +# lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines() +# lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines() +# assert len(lines_tpl) - 1 == len(lines_in) + +# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) +# pst.control_data.noptmax = -1 +# pst.pestpp_options["fill_tpl_zeros"] = True +# pst.write(os.path.join(t_d,"pest_fill.pst")) +# pyemu.os_utils.run("{0} pest_fill.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) +# obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"]) +# obf_df.index = obf_df.obsnme +# pst = pyemu.Pst(os.path.join(t_d,"pest_fill.pst")) +# res_df = pst.res - d = (obf_df.obsval - res_df.modelled).apply(np.abs) - #print(d) - print(d.max()) - assert d.max() < 1.0e-5, d - - jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest_fill.jcb")).to_dataframe().apply(np.abs) - assert jco.sum().sum() == 0, jco.sum() - - pst.control_data.noptmax = 0 - pst.write(os.path.join(t_d,"pest.pst")) - pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) - - # check the input file - the last two number should be the same - arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref")) - assert arr[-2] == arr[-1] - - lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines() - lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines() - assert len(lines_tpl) - 1 == len(lines_in) - - pst = pyemu.Pst(os.path.join(os.path.join(model_d,"template"), "pest.pst")) - #pst.drop_observations(os.path.join(t_d,"AOC_obs.txt.ins"),pst_path=".") - dum_obs = ['h01_03', 'h01_07', 'dummy_obs'] - pst.observation_data.drop(index=dum_obs, inplace=True) - pst.model_output_data = pd.DataFrame({"pest_file":"out1dum.dat.ins", - "model_file":'out1.dat'},index=["out1dum.dat.ins"]) - #pst.instruction_files = ['out1dum.dat.ins'] - pst.control_data.noptmax = -1 - pst.write(os.path.join(t_d, "pest_dum.pst")) - pyemu.os_utils.run("{0} pest_dum.pst".format(exe_path.replace("-ies", "-glm")), cwd=t_d) - obf_df = pd.read_csv(os.path.join(t_d, "out1.dat.obf"), delim_whitespace=True, header=None, - names=["obsnme", "obsval"]) - obf_df.index = obf_df.obsnme - pst = pyemu.Pst(os.path.join(t_d, "pest_dum.pst")) - res_df = pst.res - - d = (obf_df.obsval - res_df.modelled).apply(np.abs) - # print(d) - print(d.max()) - assert d.max() < 1.0e-5, d - - jco = pyemu.Jco.from_binary(os.path.join(t_d, "pest_dum.jcb")).to_dataframe().apply(np.abs) - assert jco.sum().sum() == 0, jco.sum() - - - -def ext_stdcol_test(): - model_d = "ies_10par_xsec" +# d = (obf_df.obsval - res_df.modelled).apply(np.abs) +# #print(d) +# print(d.max()) +# assert d.max() < 1.0e-5, d + +# jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest_fill.jcb")).to_dataframe().apply(np.abs) +# assert jco.sum().sum() == 0, jco.sum() + +# pst.control_data.noptmax = 0 +# pst.write(os.path.join(t_d,"pest.pst")) +# pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) + +# # check the input file - the last two number should be the same +# arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref")) +# assert arr[-2] == arr[-1] + +# lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines() +# lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines() +# assert len(lines_tpl) - 1 == len(lines_in) + +# pst = pyemu.Pst(os.path.join(os.path.join(model_d,"template"), "pest.pst")) +# #pst.drop_observations(os.path.join(t_d,"AOC_obs.txt.ins"),pst_path=".") +# dum_obs = ['h01_03', 'h01_07', 'dummy_obs'] +# pst.observation_data.drop(index=dum_obs, inplace=True) +# pst.model_output_data = pd.DataFrame({"pest_file":"out1dum.dat.ins", +# "model_file":'out1.dat'},index=["out1dum.dat.ins"]) +# #pst.instruction_files = ['out1dum.dat.ins'] +# pst.control_data.noptmax = -1 +# pst.write(os.path.join(t_d, "pest_dum.pst")) +# pyemu.os_utils.run("{0} pest_dum.pst".format(exe_path.replace("-ies", "-glm")), cwd=t_d) +# obf_df = pd.read_csv(os.path.join(t_d, "out1.dat.obf"), delim_whitespace=True, header=None, +# names=["obsnme", "obsval"]) +# obf_df.index = obf_df.obsnme +# pst = pyemu.Pst(os.path.join(t_d, "pest_dum.pst")) +# res_df = pst.res + +# d = (obf_df.obsval - res_df.modelled).apply(np.abs) +# # print(d) +# print(d.max()) +# assert d.max() < 1.0e-5, d + +# jco = pyemu.Jco.from_binary(os.path.join(t_d, "pest_dum.jcb")).to_dataframe().apply(np.abs) +# assert jco.sum().sum() == 0, jco.sum() + + + +# def ext_stdcol_test(): +# model_d = "ies_10par_xsec" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_ext_stdcol") - if os.path.exists(m_d): - shutil.rmtree(m_d) - shutil.copytree(t_d,m_d) +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_ext_stdcol") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# shutil.copytree(t_d,m_d) - pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) - obs = pst.observation_data - obs.loc[pst.nnz_obs_names,"standard_deviation"] = 1/obs.loc[pst.nnz_obs_names,"weight"] - pst.add_transform_columns() - par = pst.parameter_data - par.loc[pst.adj_par_names,"standard_deviation"] = (par.loc[pst.adj_par_names,"parubnd_trans"] - par.loc[pst.adj_par_names,"parlbnd_trans"]) / 4.0 - #par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"] - pst.pestpp_options["ies_num_reals"] = 10 - pst.control_data.noptmax = -1 - pst.write(os.path.join(m_d,"pest_base.pst")) - pyemu.os_utils.run("{0} pest_base.pst".format(exe_path),cwd=m_d) - - pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) - pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) - df1 = pd.read_csv(os.path.join(m_d,"pest_base.phi.meas.csv"),index_col=0) - df2 = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.phi.meas.csv"),index_col=0) - - d = (df1 - df2).apply(np.abs) - print(d.max()) - assert d.max().max() < 1.0e-6,d.max().max() - - pst.pestpp_options["ies_num_reals"] = 100000 - pst.control_data.noptmax = -2 - obs = pst.observation_data - obs.loc[pst.nnz_obs_names,"standard_deviation"] = 7.5 - pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) - pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) - df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names] - d = (df.std() - obs.loc[pst.nnz_obs_names,"standard_deviation"]).apply(np.abs) - print(d) - assert d.max() < 0.1,d.max() +# pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) +# obs = pst.observation_data +# obs.loc[pst.nnz_obs_names,"standard_deviation"] = 1/obs.loc[pst.nnz_obs_names,"weight"] +# pst.add_transform_columns() +# par = pst.parameter_data +# par.loc[pst.adj_par_names,"standard_deviation"] = (par.loc[pst.adj_par_names,"parubnd_trans"] - par.loc[pst.adj_par_names,"parlbnd_trans"]) / 4.0 +# #par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"] +# pst.pestpp_options["ies_num_reals"] = 10 +# pst.control_data.noptmax = -1 +# pst.write(os.path.join(m_d,"pest_base.pst")) +# pyemu.os_utils.run("{0} pest_base.pst".format(exe_path),cwd=m_d) + +# pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) +# pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) +# df1 = pd.read_csv(os.path.join(m_d,"pest_base.phi.meas.csv"),index_col=0) +# df2 = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.phi.meas.csv"),index_col=0) + +# d = (df1 - df2).apply(np.abs) +# print(d.max()) +# assert d.max().max() < 1.0e-6,d.max().max() + +# pst.pestpp_options["ies_num_reals"] = 100000 +# pst.control_data.noptmax = -2 +# obs = pst.observation_data +# obs.loc[pst.nnz_obs_names,"standard_deviation"] = 7.5 +# pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) +# pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) +# df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names] +# d = (df.std() - obs.loc[pst.nnz_obs_names,"standard_deviation"]).apply(np.abs) +# print(d) +# assert d.max() < 0.1,d.max() - obs = pst.observation_data - obs.loc[pst.nnz_obs_names,"upper_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 1.1 - obs.loc[pst.nnz_obs_names,"lower_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 0.9 - par = pst.parameter_data - par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"] - pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) - pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) - df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names] - mn = df.min() - mx = df.max() - dmn = mn - obs.loc[pst.nnz_obs_names,"obsval"] * 0.9 - print(obs.loc[pst.nnz_obs_names,"obsval"] * 0.9) - print(mn) - print(dmn) - dmx = mx - obs.loc[pst.nnz_obs_names,"obsval"] * 1.1 - print(obs.loc[pst.nnz_obs_names,"obsval"] * 1.1) - print(mx) - print(dmx) - - dmn = dmn.apply(np.abs) - dmx = dmx.apply(np.abs) - - assert dmn.max() < 1.0e-6,dmn - assert dmx.max() < 1.0e-6,dmx - - -def mf6_v5_ies_test(): - model_d = "mf6_freyberg" - - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_ies_glm_loc") - #if os.path.exists(m_d): - # shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst")) - pst.control_data.noptmax = 0 - pst.write(os.path.join(t_d,"freyberg6_run_ies.pst")) - pyemu.os_utils.run("{0} freyberg6_run_ies.pst".format(exe_path),cwd=t_d) - - pst.control_data.noptmax = 3 - par = pst.parameter_data - - eff_lb = (par.parlbnd + (np.abs(par.parlbnd.values)*.01)).to_dict() - eff_ub = (par.parubnd - (np.abs(par.parlbnd.values)*.01)).to_dict() - log_idx = par.partrans.apply(lambda x: x=="log").to_dict() - for p,log in log_idx.items(): - if log: - lb = np.log10(par.loc[p,"parlbnd"]) - eff_lb[p] = (lb + (np.abs(lb)*.01)) - ub = np.log10(par.loc[p,"parubnd"]) - eff_ub[p] = (ub - (np.abs(ub)*.01)) - - pargp_map = par.groupby(par.pargp).groups - print(pargp_map) +# obs = pst.observation_data +# obs.loc[pst.nnz_obs_names,"upper_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 1.1 +# obs.loc[pst.nnz_obs_names,"lower_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 0.9 +# par = pst.parameter_data +# par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"] +# pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) +# pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) +# df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names] +# mn = df.min() +# mx = df.max() +# dmn = mn - obs.loc[pst.nnz_obs_names,"obsval"] * 0.9 +# print(obs.loc[pst.nnz_obs_names,"obsval"] * 0.9) +# print(mn) +# print(dmn) +# dmx = mx - obs.loc[pst.nnz_obs_names,"obsval"] * 1.1 +# print(obs.loc[pst.nnz_obs_names,"obsval"] * 1.1) +# print(mx) +# print(dmx) + +# dmn = dmn.apply(np.abs) +# dmx = dmx.apply(np.abs) + +# assert dmn.max() < 1.0e-6,dmn +# assert dmx.max() < 1.0e-6,dmx + + +# def mf6_v5_ies_test(): +# model_d = "mf6_freyberg" + +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_ies_glm_loc") +# #if os.path.exists(m_d): +# # shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst")) +# pst.control_data.noptmax = 0 +# pst.write(os.path.join(t_d,"freyberg6_run_ies.pst")) +# pyemu.os_utils.run("{0} freyberg6_run_ies.pst".format(exe_path),cwd=t_d) + +# pst.control_data.noptmax = 3 +# par = pst.parameter_data + +# eff_lb = (par.parlbnd + (np.abs(par.parlbnd.values)*.01)).to_dict() +# eff_ub = (par.parubnd - (np.abs(par.parlbnd.values)*.01)).to_dict() +# log_idx = par.partrans.apply(lambda x: x=="log").to_dict() +# for p,log in log_idx.items(): +# if log: +# lb = np.log10(par.loc[p,"parlbnd"]) +# eff_lb[p] = (lb + (np.abs(lb)*.01)) +# ub = np.log10(par.loc[p,"parubnd"]) +# eff_ub[p] = (ub - (np.abs(ub)*.01)) + +# pargp_map = par.groupby(par.pargp).groups +# print(pargp_map) - m_d = os.path.join(model_d, "master_ies_glm_noloc_standard") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.pestpp_options.pop("ies_localizer",None) - pst.pestpp_options.pop("ies_autoadaloc",None) - pst.pestpp_options["ies_bad_phi_sigma"] = 2.5 - pst.pestpp_options["ies_num_reals"] = 30 - pst.pestpp_options["ensemble_output_precision"] = 40 - pst.control_data.noptmax = 3 - pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" - pst.write(os.path.join(t_d, pst_name)) - pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, - master_dir=m_d, worker_root=model_d, port=port) - phidf = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".phi.actual.csv"))) - assert phidf.shape[0] == pst.control_data.noptmax + 1 - for i in range(1,pst.control_data.noptmax+1): - pcs = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.pcs.csv".format(i))),index_col=0) - #print(pcs) - pe = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.par.csv".format(i))),index_col=0) - print(pe.shape) - #print(pe) - groups = pcs.index.values.copy() - groups.sort() - for group in groups: - pnames = pargp_map[group].values - lb_count,ub_count = 0,0 - for pname in pnames: - lb,ub = eff_lb[pname],eff_ub[pname] - v = pe.loc[:,pname].values.copy() - if log_idx[pname]: - v = np.log10(v) - low = np.zeros_like(v,dtype=int) - low[v < lb] = 1 - high = np.zeros_like(v,dtype=int) - high[v > ub] = 1 - lb_count += low.sum() - ub_count += high.sum() - print(i,group,len(pnames),lb_count,pcs.loc[group,"num_at_near_lbound"],ub_count,pcs.loc[group,"num_at_near_ubound"]) - assert lb_count == pcs.loc[group,"num_at_near_lbound"] - assert ub_count == pcs.loc[group,"num_at_near_ubound"] +# m_d = os.path.join(model_d, "master_ies_glm_noloc_standard") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) +# pst.pestpp_options.pop("ies_localizer",None) +# pst.pestpp_options.pop("ies_autoadaloc",None) +# pst.pestpp_options["ies_bad_phi_sigma"] = 2.5 +# pst.pestpp_options["ies_num_reals"] = 30 +# pst.pestpp_options["ensemble_output_precision"] = 40 +# pst.control_data.noptmax = 3 +# pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" +# pst.write(os.path.join(t_d, pst_name)) +# pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, +# master_dir=m_d, worker_root=model_d, port=port) +# phidf = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".phi.actual.csv"))) +# assert phidf.shape[0] == pst.control_data.noptmax + 1 +# for i in range(1,pst.control_data.noptmax+1): +# pcs = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.pcs.csv".format(i))),index_col=0) +# #print(pcs) +# pe = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.par.csv".format(i))),index_col=0) +# print(pe.shape) +# #print(pe) +# groups = pcs.index.values.copy() +# groups.sort() +# for group in groups: +# pnames = pargp_map[group].values +# lb_count,ub_count = 0,0 +# for pname in pnames: +# lb,ub = eff_lb[pname],eff_ub[pname] +# v = pe.loc[:,pname].values.copy() +# if log_idx[pname]: +# v = np.log10(v) +# low = np.zeros_like(v,dtype=int) +# low[v < lb] = 1 +# high = np.zeros_like(v,dtype=int) +# high[v > ub] = 1 +# lb_count += low.sum() +# ub_count += high.sum() +# print(i,group,len(pnames),lb_count,pcs.loc[group,"num_at_near_lbound"],ub_count,pcs.loc[group,"num_at_near_ubound"]) +# assert lb_count == pcs.loc[group,"num_at_near_lbound"] +# assert ub_count == pcs.loc[group,"num_at_near_ubound"] - pst.write(os.path.join(t_d,"freyberg6_run_ies_glm_loc.pst")) - - m_d = os.path.join(model_d, "master_ies_glm_covloc") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst.pestpp_options["ies_loc_type"] = "cov" - pst.pestpp_options["ies_lambda_mults"] = [1.0] - pst.pestpp_options["lambda_scale_fac"] = [1.0] - pst.control_data.noptmax = 2 - #pst.pestpp_options.pop("ies_localizer",None) - pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_covloc.pst")) - pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_covloc.pst", num_workers=15, - master_dir=m_d, worker_root=model_d, port=port) - - m_d = os.path.join(model_d, "master_ies_glm_noloc") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.pestpp_options.pop("ies_localizer",None) - pst.pestpp_options.pop("ies_autoadaloc",None) - pst.pestpp_options["ies_lambda_mults"] = [1.0] - pst.pestpp_options["lambda_scale_fac"] = [1.0] - pst.control_data.noptmax = 2 - pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc.pst")) - pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc.pst", num_workers=15, - master_dir=m_d, worker_root=model_d, port=port) - - m_d = os.path.join(model_d, "master_ies_mda_loc") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.pestpp_options["ies_lambda_mults"] = [1.0] - pst.pestpp_options["lambda_scale_fac"] = [1.0] - pst.control_data.noptmax = 2 - pst.pestpp_options["ies_use_mda"] = True - pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_loc.pst")) - pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_loc.pst", num_workers=15, - master_dir=m_d, worker_root=model_d, port=port) +# pst.write(os.path.join(t_d,"freyberg6_run_ies_glm_loc.pst")) + +# m_d = os.path.join(model_d, "master_ies_glm_covloc") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst.pestpp_options["ies_loc_type"] = "cov" +# pst.pestpp_options["ies_lambda_mults"] = [1.0] +# pst.pestpp_options["lambda_scale_fac"] = [1.0] +# pst.control_data.noptmax = 2 +# #pst.pestpp_options.pop("ies_localizer",None) +# pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_covloc.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_covloc.pst", num_workers=15, +# master_dir=m_d, worker_root=model_d, port=port) + +# m_d = os.path.join(model_d, "master_ies_glm_noloc") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) +# pst.pestpp_options.pop("ies_localizer",None) +# pst.pestpp_options.pop("ies_autoadaloc",None) +# pst.pestpp_options["ies_lambda_mults"] = [1.0] +# pst.pestpp_options["lambda_scale_fac"] = [1.0] +# pst.control_data.noptmax = 2 +# pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc.pst", num_workers=15, +# master_dir=m_d, worker_root=model_d, port=port) + +# m_d = os.path.join(model_d, "master_ies_mda_loc") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) +# pst.pestpp_options["ies_lambda_mults"] = [1.0] +# pst.pestpp_options["lambda_scale_fac"] = [1.0] +# pst.control_data.noptmax = 2 +# pst.pestpp_options["ies_use_mda"] = True +# pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_loc.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_loc.pst", num_workers=15, +# master_dir=m_d, worker_root=model_d, port=port) - # m_d = os.path.join(model_d, "master_ies_mda_covloc") - # if os.path.exists(m_d): - # shutil.rmtree(m_d) - # pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - # pst.control_data.noptmax = 3 - # pst.pestpp_options["ies_use_mda"] = True - # pst.pestpp_options["ies_loc_type"] = "cov" - # pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_covloc.pst")) - # pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_covloc.pst", num_workers=15, - # master_dir=m_d, worker_root=model_d, port=port) +# # m_d = os.path.join(model_d, "master_ies_mda_covloc") +# # if os.path.exists(m_d): +# # shutil.rmtree(m_d) +# # pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) +# # pst.control_data.noptmax = 3 +# # pst.pestpp_options["ies_use_mda"] = True +# # pst.pestpp_options["ies_loc_type"] = "cov" +# # pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_covloc.pst")) +# # pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_covloc.pst", num_workers=15, +# # master_dir=m_d, worker_root=model_d, port=port) - m_d = os.path.join(model_d, "master_ies_mda_noloc") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.control_data.noptmax = 2 - pst.pestpp_options["ies_lambda_mults"] = [1.0] - pst.pestpp_options["lambda_scale_fac"] = [1.0] - pst.pestpp_options["ies_use_mda"] = True - pst.pestpp_options.pop("ies_localizer", None) - pst.pestpp_options.pop("ies_autoadaloc", None) - pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_noloc.pst")) - pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_noloc.pst", num_workers=15, - master_dir=m_d, worker_root=model_d, port=port) - - m_d = os.path.join(model_d, "master_ies_glm_loc_mm") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.control_data.noptmax = 2 - pst.pestpp_options["ies_lambda_mults"] = [1.0] - pst.pestpp_options["lambda_scale_fac"] = [1.0] - pst.pestpp_options["ies_num_threads"] = 1 - pst.pestpp_options["ies_use_mda"] = False - pst.pestpp_options.pop("ies_localizer", None) - pst.pestpp_options.pop("ies_autoadaloc", None) - pst.pestpp_options["ies_multimodal_alpha"] = 0.1 - pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_loc_mm.pst")) - pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_loc_mm.pst", num_workers=15, - master_dir=m_d, worker_root=model_d, port=port) - - m_d = os.path.join(model_d, "master_ies_glm_noloc_mm") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) - pst.control_data.noptmax = 2 - pst.pestpp_options["ies_use_mda"] = False - pst.pestpp_options["ies_lambda_mults"] = [1.0] - pst.pestpp_options["lambda_scale_fac"] = [1.0] - pst.pestpp_options.pop("ies_localizer", None) - pst.pestpp_options.pop("ies_autoadaloc", None) - pst.pestpp_options["ies_multimodal_alpha"] = 0.25 - pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc_mm.pst")) - pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc_mm.pst", num_workers=15, - master_dir=m_d, worker_root=model_d, port=port) - - - - -def mf6_v5_sen_test(): - - model_d = "mf6_freyberg" - - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_sen") - #if os.path.exists(m_d): - # shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_sen.pst")) - pst.pestpp_options["panther_transfer_on_finish"] = ["freyberg6_freyberg.cbc","freyberg6.lst","ies_prior.jcb"] - pst.write(os.path.join(t_d,"freyberg6_run_sen_trn.pst")) - m_d = os.path.join(model_d,"master_sen") - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "freyberg6_run_sen_trn.pst", - num_workers=15, worker_root=model_d, - port=4004,verbose=True,master_dir=m_d) - - pst = pyemu.Pst(os.path.join(m_d,"freyberg6_run_sen_trn.pst")) - mio_file = os.path.join(m_d,"freyberg6_run_sen_trn.mio") - assert os.path.exists(mio_file),mio_file - df = pd.read_csv(mio_file) - assert df.shape[0] > 1 - msn_file = mio_file.replace(".mio",".msn") - assert os.path.exists(msn_file),msn_file - msngrp_file = msn_file.replace(".msn",".group.msn") - assert os.path.exists(msngrp_file),msngrp_file - - jcb_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".jcb")] - print(len(jcb_files)) - assert len(jcb_files) == 52 - for jcb_file in jcb_files: - j = pyemu.Jco.from_binary(os.path.join(m_d,jcb_file)) - - lst_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".lst")] - print(len(lst_files)) - assert len(lst_files) == 52 +# m_d = os.path.join(model_d, "master_ies_mda_noloc") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) +# pst.control_data.noptmax = 2 +# pst.pestpp_options["ies_lambda_mults"] = [1.0] +# pst.pestpp_options["lambda_scale_fac"] = [1.0] +# pst.pestpp_options["ies_use_mda"] = True +# pst.pestpp_options.pop("ies_localizer", None) +# pst.pestpp_options.pop("ies_autoadaloc", None) +# pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_noloc.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_noloc.pst", num_workers=15, +# master_dir=m_d, worker_root=model_d, port=port) + +# m_d = os.path.join(model_d, "master_ies_glm_loc_mm") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) +# pst.control_data.noptmax = 2 +# pst.pestpp_options["ies_lambda_mults"] = [1.0] +# pst.pestpp_options["lambda_scale_fac"] = [1.0] +# pst.pestpp_options["ies_num_threads"] = 1 +# pst.pestpp_options["ies_use_mda"] = False +# pst.pestpp_options.pop("ies_localizer", None) +# pst.pestpp_options.pop("ies_autoadaloc", None) +# pst.pestpp_options["ies_multimodal_alpha"] = 0.1 +# pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_loc_mm.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_loc_mm.pst", num_workers=15, +# master_dir=m_d, worker_root=model_d, port=port) + +# m_d = os.path.join(model_d, "master_ies_glm_noloc_mm") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) +# pst.control_data.noptmax = 2 +# pst.pestpp_options["ies_use_mda"] = False +# pst.pestpp_options["ies_lambda_mults"] = [1.0] +# pst.pestpp_options["lambda_scale_fac"] = [1.0] +# pst.pestpp_options.pop("ies_localizer", None) +# pst.pestpp_options.pop("ies_autoadaloc", None) +# pst.pestpp_options["ies_multimodal_alpha"] = 0.25 +# pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc_mm.pst")) +# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc_mm.pst", num_workers=15, +# master_dir=m_d, worker_root=model_d, port=port) + + + + +# def mf6_v5_sen_test(): + +# model_d = "mf6_freyberg" + +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_sen") +# #if os.path.exists(m_d): +# # shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_sen.pst")) +# pst.pestpp_options["panther_transfer_on_finish"] = ["freyberg6_freyberg.cbc","freyberg6.lst","ies_prior.jcb"] +# pst.write(os.path.join(t_d,"freyberg6_run_sen_trn.pst")) +# m_d = os.path.join(model_d,"master_sen") +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "freyberg6_run_sen_trn.pst", +# num_workers=15, worker_root=model_d, +# port=4004,verbose=True,master_dir=m_d) + +# pst = pyemu.Pst(os.path.join(m_d,"freyberg6_run_sen_trn.pst")) +# mio_file = os.path.join(m_d,"freyberg6_run_sen_trn.mio") +# assert os.path.exists(mio_file),mio_file +# df = pd.read_csv(mio_file) +# assert df.shape[0] > 1 +# msn_file = mio_file.replace(".mio",".msn") +# assert os.path.exists(msn_file),msn_file +# msngrp_file = msn_file.replace(".msn",".group.msn") +# assert os.path.exists(msngrp_file),msngrp_file + +# jcb_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".jcb")] +# print(len(jcb_files)) +# assert len(jcb_files) == 52 +# for jcb_file in jcb_files: +# j = pyemu.Jco.from_binary(os.path.join(m_d,jcb_file)) + +# lst_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".lst")] +# print(len(lst_files)) +# assert len(lst_files) == 52 - cbc_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx") and f.lower().endswith(".cbc")] - print(len(cbc_files)) - assert len(cbc_files) == 52 +# cbc_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx") and f.lower().endswith(".cbc")] +# print(len(cbc_files)) +# assert len(cbc_files) == 52 -def mf6_v5_opt_stack_test(): - model_d = "mf6_freyberg" +# def mf6_v5_opt_stack_test(): +# model_d = "mf6_freyberg" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_opt_stack") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_opt.pst")) - m_d = os.path.join(model_d,"master_opt_stack") - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-opt"), "freyberg6_run_opt.pst", - num_workers=15, master_dir=m_d,worker_root=model_d, - port=port) - - assert os.path.exists(os.path.join(m_d,"freyberg6_run_opt.1.sim+chance.rei")) - assert os.path.exists(os.path.join(m_d,"freyberg6_run_opt.1.obs_stack.csv")) - - -def mf6_v5_glm_test(): - model_d = "mf6_freyberg" +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_opt_stack") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_opt.pst")) +# m_d = os.path.join(model_d,"master_opt_stack") +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-opt"), "freyberg6_run_opt.pst", +# num_workers=15, master_dir=m_d,worker_root=model_d, +# port=port) + +# assert os.path.exists(os.path.join(m_d,"freyberg6_run_opt.1.sim+chance.rei")) +# assert os.path.exists(os.path.join(m_d,"freyberg6_run_opt.1.obs_stack.csv")) + + +# def mf6_v5_glm_test(): +# model_d = "mf6_freyberg" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_glm") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_glm.pst")) - m_d = os.path.join(model_d,"master_glm") - pyemu.os_utils.start_workers(t_d, "pestpp-glm", "freyberg6_run_glm.pst", - num_workers=15, master_dir=m_d,worker_root=model_d, - port=port) - - oe_file = os.path.join(m_d,"freyberg6_run_glm.post.obsen.csv") - assert os.path.exists(oe_file) - oe = pd.read_csv(oe_file) - assert oe.shape[0] == pst.pestpp_options["glm_num_reals"],"{0},{1}".\ - format(oe.shape[0],pst.pestpp_options["glm_num_reals"]) - - -def cmdline_test(): - model_d = "mf6_freyberg" +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_glm") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_glm.pst")) +# m_d = os.path.join(model_d,"master_glm") +# pyemu.os_utils.start_workers(t_d, "pestpp-glm", "freyberg6_run_glm.pst", +# num_workers=15, master_dir=m_d,worker_root=model_d, +# port=port) + +# oe_file = os.path.join(m_d,"freyberg6_run_glm.post.obsen.csv") +# assert os.path.exists(oe_file) +# oe = pd.read_csv(oe_file) +# assert oe.shape[0] == pst.pestpp_options["glm_num_reals"],"{0},{1}".\ +# format(oe.shape[0],pst.pestpp_options["glm_num_reals"]) + + +# def cmdline_test(): +# model_d = "mf6_freyberg" - t_d = os.path.join(model_d,"template") - pst_name = "freyberg6_run_glm.pst" - pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_glm.pst")) - pst.pestpp_options["debug_parse_only"] = True - pst_name = "CmdLine_test.pst" #camel case on purpose for linux testing - pst.write(os.path.join(t_d,pst_name)) - pyemu.os_utils.run("{0} {1}".format(exe_path,pst_name),cwd=t_d) - pyemu.os_utils.run("{0} {1} /h :4004".format(exe_path,pst_name),cwd=t_d) - pyemu.os_utils.run("{0} {1} /r /h :4004".format(exe_path.replace("-ies","-glm"),pst_name),cwd=t_d) - pyemu.os_utils.run("{0} {1} /r ".format(exe_path.replace("-ies","-glm"),pst_name),cwd=t_d) +# t_d = os.path.join(model_d,"template") +# pst_name = "freyberg6_run_glm.pst" +# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_glm.pst")) +# pst.pestpp_options["debug_parse_only"] = True +# pst_name = "CmdLine_test.pst" #camel case on purpose for linux testing +# pst.write(os.path.join(t_d,pst_name)) +# pyemu.os_utils.run("{0} {1}".format(exe_path,pst_name),cwd=t_d) +# pyemu.os_utils.run("{0} {1} /h :4004".format(exe_path,pst_name),cwd=t_d) +# pyemu.os_utils.run("{0} {1} /r /h :4004".format(exe_path.replace("-ies","-glm"),pst_name),cwd=t_d) +# pyemu.os_utils.run("{0} {1} /r ".format(exe_path.replace("-ies","-glm"),pst_name),cwd=t_d) - try: - pyemu.os_utils.run("{0} {1} \\h :4004".format(exe_path,pst_name),cwd=t_d) +# try: +# pyemu.os_utils.run("{0} {1} \\h :4004".format(exe_path,pst_name),cwd=t_d) - except: - pass - else: - raise Exception("should have failed") +# except: +# pass +# else: +# raise Exception("should have failed") - try: - pyemu.os_utils.run("{0} {1} :4004".format(exe_path,pst_name),cwd=t_d) +# try: +# pyemu.os_utils.run("{0} {1} :4004".format(exe_path,pst_name),cwd=t_d) - except: - pass - else: - raise Exception("should have failed") +# except: +# pass +# else: +# raise Exception("should have failed") - try: - pyemu.os_utils.run("{0} {1} /h 4004".format(exe_path,pst_name),cwd=t_d) +# try: +# pyemu.os_utils.run("{0} {1} /h 4004".format(exe_path,pst_name),cwd=t_d) - except: - pass - else: - raise Exception("should have failed") +# except: +# pass +# else: +# raise Exception("should have failed") -def fr_fail_test(): - model_d = "ies_10par_xsec" - base_d = os.path.join(model_d, "template") - new_d = os.path.join(model_d, "test_template") - if os.path.exists(new_d): - shutil.rmtree(new_d) - shutil.copytree(base_d, new_d) - print(platform.platform().lower()) - pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) - with open(os.path.join(new_d,"run.py"),'w') as f: - f.write("import pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\nprint(junk)\n") - pst.model_command = "python run.py" - oe_file = os.path.join(new_d, "pest.0.obs.csv") - if os.path.exists(oe_file): - os.remove(oe_file) - pst.control_data.noptmax = 1 - pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list" - pst.pestpp_options["ies_num_reals"] = 10 - #pst.pestpp_options["panther_agent_freeze_on_fail"] = True - pst.write(os.path.join(new_d, "pest.pst")) - try: - pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d) - except: - pass - else: - raise Exception("should have failed") - - assert not os.path.exists(oe_file) - m_d = os.path.join(model_d,"fr_fail_master") - try: - pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d) - except: - pass - else: - raise Exception("should have failed") - oe_file = os.path.join(m_d, "pest.0.obs.csv") - assert not os.path.exists(oe_file) - - trx_files = [f for f in os.listdir(m_d) if f.endswith(".list")] - print(trx_files) - assert len(trx_files) == 11,len(trx_files) - - - - - -def sen_grp_test(): +# def fr_fail_test(): +# model_d = "ies_10par_xsec" +# base_d = os.path.join(model_d, "template") +# new_d = os.path.join(model_d, "test_template") +# if os.path.exists(new_d): +# shutil.rmtree(new_d) +# shutil.copytree(base_d, new_d) +# print(platform.platform().lower()) +# pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) +# with open(os.path.join(new_d,"run.py"),'w') as f: +# f.write("import pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\nprint(junk)\n") +# pst.model_command = "python run.py" +# oe_file = os.path.join(new_d, "pest.0.obs.csv") +# if os.path.exists(oe_file): +# os.remove(oe_file) +# pst.control_data.noptmax = 1 +# pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list" +# pst.pestpp_options["ies_num_reals"] = 10 +# #pst.pestpp_options["panther_agent_freeze_on_fail"] = True +# pst.write(os.path.join(new_d, "pest.pst")) +# try: +# pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d) +# except: +# pass +# else: +# raise Exception("should have failed") + +# assert not os.path.exists(oe_file) +# m_d = os.path.join(model_d,"fr_fail_master") +# try: +# pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d) +# except: +# pass +# else: +# raise Exception("should have failed") +# oe_file = os.path.join(m_d, "pest.0.obs.csv") +# assert not os.path.exists(oe_file) + +# trx_files = [f for f in os.listdir(m_d) if f.endswith(".list")] +# print(trx_files) +# assert len(trx_files) == 11,len(trx_files) + + + + + +# def sen_grp_test(): - model_d = "ies_10par_xsec" +# model_d = "ies_10par_xsec" - t_d = os.path.join(model_d, "template") - m_d = os.path.join(model_d, "master_sen_group") - if os.path.exists(m_d): - shutil.rmtree(m_d) +# t_d = os.path.join(model_d, "template") +# m_d = os.path.join(model_d, "master_sen_group") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-sen"), "pest.pst", 10, master_dir=m_d, - worker_root=model_d, port=port) - - msn_file = os.path.join(m_d,"pest.msn") - msndf = pd.read_csv(msn_file) - - grp_file = msn_file.replace(".msn",".group.msn") - grpdf = pd.read_csv(grp_file) - assert msndf.shape[0] == grpdf.shape[0] - for col in ["sen_mean","sen_mean_abs","sen_std_dev"]: - diff = np.abs(msndf.loc[:,col].sum() - grpdf.loc[:,col].sum()) - print(col,diff) - assert diff < 1.0e-6 - - -def agnostic_path_test(): - model_d = "ies_10par_xsec" - - t_d = os.path.join(model_d, "template") - m_d = os.path.join(model_d, "test_path") - if os.path.exists(m_d): - shutil.rmtree(m_d) - shutil.copytree(t_d,m_d) - - pst = pyemu.Pst(os.path.join(m_d, "pest.pst")) - pst.parameter_data.loc[pst.adj_par_names,"parval1"] = np.random.random(pst.npar_adj) - pst.control_data.noptmax = 0 - pst.write(os.path.join(m_d,"pest.pst")) - pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=m_d) - - arr1 = np.loadtxt(os.path.join(m_d,"hk_Layer_1.ref")) - arr2 = np.loadtxt(os.path.join(m_d,"nested","really","deep","hk_Layer_1.ref")) - d = np.abs(arr1-arr2).sum() - print(d) - assert d == 0,d - -def fr_timeout_test(): - model_d = "ies_10par_xsec" - base_d = os.path.join(model_d, "template") - new_d = os.path.join(model_d, "test_template") - if os.path.exists(new_d): - shutil.rmtree(new_d) - shutil.copytree(base_d, new_d) - print(platform.platform().lower()) - pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) - with open(os.path.join(new_d,"run.py"),'w') as f: - f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n") - f.write("if not os.path.exists('run.info'):\n exit()\n") - f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n") - f.write("if rnum % 2 == 0:\n time.sleep(10000000)\n") - pst.model_command = "python run.py" - oe_file = os.path.join(new_d, "pest.0.obs.csv") - if os.path.exists(oe_file): - os.remove(oe_file) - pst.control_data.noptmax = -1 - pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10 - pst.pestpp_options["overdue_giveup_minutes"] = 0.25 - pst.pestpp_options["ies_num_reals"] = 10 - pst.pestpp_options["ies_include_base"] = False - pst.pestpp_options["max_run_fail"] = 1 - - #pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list" - pst.pestpp_options["panther_agent_freeze_on_fail"] = False - pst.write(os.path.join(new_d, "pest.pst")) - - m_d = os.path.join(model_d,"fr_timeout_master") - pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d) - oe_file = os.path.join(m_d, "pest.0.obs.csv") - assert os.path.exists(oe_file) - oe = pd.read_csv(oe_file,index_col=0) - print(oe.shape) - assert oe.shape[0] == 5,oe.shape - - with open(os.path.join(new_d,"run.py"),'w') as f: - f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n") - f.write("if not os.path.exists('run.info'):\n exit()\n") - f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n") - f.write("if rnum % 10 == 0:\n print(junk)\n") - pst.pestpp_options = {} - pst.pestpp_options["ies_num_reals"] = 20 # hard coded to conditional below - pst.pestpp_options["panther_agent_freeze_on_fail"] = True - #pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10 - #pst.pestpp_options["overdue_giveup_minutes"] = 0.25 - pst.write(os.path.join(new_d, "pest.pst")) - pst.control_data.noptmax = 2 - - pst.write(os.path.join(new_d, "pest.pst")) - m_d = os.path.join(model_d,"fr_timeout_master_freeze") - #num workers hard coded with conditional below - pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=10,worker_root=model_d,master_dir=m_d) - #df = pyemu.helpers.parse_rmr_file(os.path.join(m_d,"pest.rmr")) - #print(df.action.to_list()) - oe = pd.read_csv(os.path.join(m_d,"pest.{0}.obs.csv".format(pst.control_data.noptmax)),index_col=0) - assert oe.shape[0] == 17 # hard coded to num reals - with open(os.path.join(m_d,"pest.rmr"),'r') as f: - for line in f: - if "timeout" in line.lower(): - raise Exception() - if line.strip().lower().endswith("agents connected"): - num = int(line.strip().split()[0]) - print(line.strip()) - assert num == 7 # hard coded above - - - -def ins_missing_e_test(): - import os - import shutil - import pyemu - t_d = os.path.join("tplins_test_1","test_missing_e") - if os.path.exists(t_d): - shutil.rmtree(t_d) - os.makedirs(t_d) - bd = os.getcwd() - os.chdir(t_d) - with open("model.output.bak",'w') as f: - f.write("12345-123\n") - pst = pyemu.helpers.pst_from_parnames_obsnames(["p1"],["o1"]) - pst.control_data.noptmax = 0 - with open("forward_run.py",'w') as f: - f.write("import shutil\n") - f.write("shutil.copy2('model.output.bak','model.output')\n") - pst.model_command = "python forward_run.py" - pst.write("test.pst") - os.chdir(bd) - try: - pyemu.os_utils.run("{0} test.pst".format(exe_path),cwd=t_d) - except: - pass - else: - raise Exception("should have failed") - - -def prep_ends(): - model_d = "mf6_freyberg" - base_d = os.path.join(model_d, "template") - new_d = os.path.join(model_d, "ends") - if os.path.exists(new_d): - shutil.rmtree(new_d) - os.makedirs(new_d) - skip = ["pst","csv","log","grb","hds","par","rei","lst","jcb","cov","rec","cbc"] - files = [f for f in os.listdir(base_d) if f.lower().split('.')[-1] not in skip] - print(files) - [shutil.copy2(os.path.join(base_d,f),os.path.join(new_d,f)) for f in files] - [shutil.copy2(os.path.join(base_d,f),os.path.join(new_d,f)) for f in ["ies_prior.jcb"]] +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-sen"), "pest.pst", 10, master_dir=m_d, +# worker_root=model_d, port=port) + +# msn_file = os.path.join(m_d,"pest.msn") +# msndf = pd.read_csv(msn_file) + +# grp_file = msn_file.replace(".msn",".group.msn") +# grpdf = pd.read_csv(grp_file) +# assert msndf.shape[0] == grpdf.shape[0] +# for col in ["sen_mean","sen_mean_abs","sen_std_dev"]: +# diff = np.abs(msndf.loc[:,col].sum() - grpdf.loc[:,col].sum()) +# print(col,diff) +# assert diff < 1.0e-6 + + +# def agnostic_path_test(): +# model_d = "ies_10par_xsec" + +# t_d = os.path.join(model_d, "template") +# m_d = os.path.join(model_d, "test_path") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# shutil.copytree(t_d,m_d) + +# pst = pyemu.Pst(os.path.join(m_d, "pest.pst")) +# pst.parameter_data.loc[pst.adj_par_names,"parval1"] = np.random.random(pst.npar_adj) +# pst.control_data.noptmax = 0 +# pst.write(os.path.join(m_d,"pest.pst")) +# pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=m_d) + +# arr1 = np.loadtxt(os.path.join(m_d,"hk_Layer_1.ref")) +# arr2 = np.loadtxt(os.path.join(m_d,"nested","really","deep","hk_Layer_1.ref")) +# d = np.abs(arr1-arr2).sum() +# print(d) +# assert d == 0,d + +# def fr_timeout_test(): +# model_d = "ies_10par_xsec" +# base_d = os.path.join(model_d, "template") +# new_d = os.path.join(model_d, "test_template") +# if os.path.exists(new_d): +# shutil.rmtree(new_d) +# shutil.copytree(base_d, new_d) +# print(platform.platform().lower()) +# pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) +# with open(os.path.join(new_d,"run.py"),'w') as f: +# f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n") +# f.write("if not os.path.exists('run.info'):\n exit()\n") +# f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n") +# f.write("if rnum % 2 == 0:\n time.sleep(10000000)\n") +# pst.model_command = "python run.py" +# oe_file = os.path.join(new_d, "pest.0.obs.csv") +# if os.path.exists(oe_file): +# os.remove(oe_file) +# pst.control_data.noptmax = -1 +# pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10 +# pst.pestpp_options["overdue_giveup_minutes"] = 0.25 +# pst.pestpp_options["ies_num_reals"] = 10 +# pst.pestpp_options["ies_include_base"] = False +# pst.pestpp_options["max_run_fail"] = 1 + +# #pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list" +# pst.pestpp_options["panther_agent_freeze_on_fail"] = False +# pst.write(os.path.join(new_d, "pest.pst")) + +# m_d = os.path.join(model_d,"fr_timeout_master") +# pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d) +# oe_file = os.path.join(m_d, "pest.0.obs.csv") +# assert os.path.exists(oe_file) +# oe = pd.read_csv(oe_file,index_col=0) +# print(oe.shape) +# assert oe.shape[0] == 5,oe.shape + +# with open(os.path.join(new_d,"run.py"),'w') as f: +# f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n") +# f.write("if not os.path.exists('run.info'):\n exit()\n") +# f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n") +# f.write("if rnum % 10 == 0:\n print(junk)\n") +# pst.pestpp_options = {} +# pst.pestpp_options["ies_num_reals"] = 20 # hard coded to conditional below +# pst.pestpp_options["panther_agent_freeze_on_fail"] = True +# #pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10 +# #pst.pestpp_options["overdue_giveup_minutes"] = 0.25 +# pst.write(os.path.join(new_d, "pest.pst")) +# pst.control_data.noptmax = 2 + +# pst.write(os.path.join(new_d, "pest.pst")) +# m_d = os.path.join(model_d,"fr_timeout_master_freeze") +# #num workers hard coded with conditional below +# pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=10,worker_root=model_d,master_dir=m_d) +# #df = pyemu.helpers.parse_rmr_file(os.path.join(m_d,"pest.rmr")) +# #print(df.action.to_list()) +# oe = pd.read_csv(os.path.join(m_d,"pest.{0}.obs.csv".format(pst.control_data.noptmax)),index_col=0) +# assert oe.shape[0] == 17 # hard coded to num reals +# with open(os.path.join(m_d,"pest.rmr"),'r') as f: +# for line in f: +# if "timeout" in line.lower(): +# raise Exception() +# if line.strip().lower().endswith("agents connected"): +# num = int(line.strip().split()[0]) +# print(line.strip()) +# assert num == 7 # hard coded above + + + +# def ins_missing_e_test(): +# import os +# import shutil +# import pyemu +# t_d = os.path.join("tplins_test_1","test_missing_e") +# if os.path.exists(t_d): +# shutil.rmtree(t_d) +# os.makedirs(t_d) +# bd = os.getcwd() +# os.chdir(t_d) +# with open("model.output.bak",'w') as f: +# f.write("12345-123\n") +# pst = pyemu.helpers.pst_from_parnames_obsnames(["p1"],["o1"]) +# pst.control_data.noptmax = 0 +# with open("forward_run.py",'w') as f: +# f.write("import shutil\n") +# f.write("shutil.copy2('model.output.bak','model.output')\n") +# pst.model_command = "python forward_run.py" +# pst.write("test.pst") +# os.chdir(bd) +# try: +# pyemu.os_utils.run("{0} test.pst".format(exe_path),cwd=t_d) +# except: +# pass +# else: +# raise Exception("should have failed") + + +# def prep_ends(): +# model_d = "mf6_freyberg" +# base_d = os.path.join(model_d, "template") +# new_d = os.path.join(model_d, "ends") +# if os.path.exists(new_d): +# shutil.rmtree(new_d) +# os.makedirs(new_d) +# skip = ["pst","csv","log","grb","hds","par","rei","lst","jcb","cov","rec","cbc"] +# files = [f for f in os.listdir(base_d) if f.lower().split('.')[-1] not in skip] +# print(files) +# [shutil.copy2(os.path.join(base_d,f),os.path.join(new_d,f)) for f in files] +# [shutil.copy2(os.path.join(base_d,f),os.path.join(new_d,f)) for f in ["ies_prior.jcb"]] - pyemu.os_utils.run("mf6",cwd=new_d) - pst = pyemu.Pst(os.path.join(base_d,"freyberg6_run_ies.pst")) - pst.control_data.noptmax = 0 - pst.pestpp_options = {} - pst.pestpp_options["ies_par_en"] = "prior.jcb" - - pst.write(os.path.join(new_d,"freyberg6_run_ies.pst"),version=2) - pyemu.os_utils.run("pestpp-ies freyberg6_run_ies.pst",cwd=new_d) - - build_and_draw_prior(new_d,num_reals=5000) - pst.control_data.noptmax = -1 - pst.write(os.path.join(new_d,"freyberg6_run_ies.pst"),version=2) - m_d = os.path.join(model_d,"ends_master") - if os.path.exists(m_d): - shutil.rmtree(m_d) - - pyemu.os_utils.start_workers(new_d,"pestpp-ies","freyberg6_run_ies.pst",num_workers=15,worker_root=model_d,master_dir=m_d) - - -def build_and_draw_prior(t_d="ends",num_reals=500): - import flopy - - sim = flopy.mf6.MFSimulation.load(sim_ws=t_d) - m = sim.get_model("freyberg6") - xgrid = m.modelgrid.xcellcenters - ygrid = m.modelgrid.ycellcenters - pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst")) - par = pst.parameter_data - static_par = par.loc[par.parnme.apply(lambda x: x[:3] in ["npf","sto"]),:].copy() - static_par.loc[:, "i"] = static_par.parnme.apply(lambda x: int(x.split('_')[3])) - static_par.loc[:, "j"] = static_par.parnme.apply(lambda x: int(x.split('_')[4])) - static_par.loc[:, "x"] = static_par.apply(lambda x: xgrid[x.i,x.j],axis=1) - static_par.loc[:, "y"] = static_par.apply(lambda x: ygrid[x.i, x.j], axis=1) - static_par.loc[:,"pargp"] = static_par.parnme.apply(lambda x: "_".join(x.split('_')[:3])) - - wel_par = par.loc[par.parnme.apply(lambda x: x.startswith("wel")),:].copy() - wel_par.loc[:,"x"] = wel_par.parnme.apply(lambda x: int(x.split('_')[-1])) - wel_par.loc[:,"y"] = 0.0 - wel_par.loc[:,"pargp"] = wel_par.parnme.apply(lambda x: '_'.join(x.split('_')[:-1])) - - rch_par = par.loc[par.parnme.str.startswith("rch"),:].copy() - rch_par.loc[:,"x"] = rch_par.parnme.apply(lambda x: int(x.split('_')[-1])) - rch_par.loc[:,"y"] = 0.0 - - spatial_v = pyemu.geostats.ExpVario(contribution=1.0,a=1000.0) - temporal_v = pyemu.geostats.ExpVario(contribution=1.0,a=3) - spatial_gs = pyemu.geostats.GeoStruct(variograms=spatial_v) - temporal_gs = pyemu.geostats.GeoStruct(variograms=temporal_v) - - static_struct_dict = {spatial_gs:[]} - sgrps = static_par.pargp.unique() - sgrps.sort() - for pargp in sgrps: - static_struct_dict[spatial_gs].append(static_par.loc[static_par.pargp==pargp,["parnme","x","y","i","j"]]) - temporal_struct_dict = {temporal_gs: [rch_par.loc[:, ["parnme", "x", "y"]]]} - wgrps = wel_par.pargp.unique() - wgrps.sort() - for pargp in wgrps: - temporal_struct_dict[temporal_gs].append(wel_par.loc[wel_par.pargp == pargp, ["parnme", "x", "y"]]) - - struct_dict = static_struct_dict - for k,v in temporal_struct_dict.items(): - struct_dict[k] = v - print(struct_dict) - np.random.seed(pyemu.en.SEED) - pe = pyemu.helpers.geostatistical_draws(pst,struct_dict=struct_dict,num_reals=num_reals) - pe.to_binary(os.path.join(t_d,"prior.jcb")) - - -def run(): - model_d = "mf6_freyberg" - t_d = os.path.join(model_d,"template") - pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" - pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, - worker_root=model_d, port=4004) - -def sweep_bin_test(): - - model_d = "ies_10par_xsec" - t_d = os.path.join(model_d,"template") - m_d = os.path.join(model_d,"master_sweep_bin") - if os.path.exists(m_d): - shutil.rmtree(m_d) - pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) - pe = pyemu.ParameterEnsemble.from_uniform_draw(pst,num_reals=50)#.loc[:,pst.par_names[:2]] - - pe.to_csv(os.path.join(t_d,"sweep_in.csv")) - pe._df.index = pe.index.map(str) - print(pe.index) - pe.to_dense(os.path.join(t_d,"sweep_in.bin")) - pst.pestpp_options["ies_par_en"] = "sweep_in.csv" - pst.pestpp_options["sweep_forgive"] = True - pst.pestpp_options["sweep_parameter_file"] = "sweep_in.bin" - pst.control_data.noptmax = -1 - pst.pestpp_options.pop("ies_num_reals",None) - pst.write(os.path.join(t_d,"pest_forgive.pst")) - pst.pestpp_options["sweep_output_file"] = "sweep_out.bin" - pst.pestpp_options["sweep_chunk"] = 9 - pst.pestpp_options["ies_include_base"] = False - pst.write(os.path.join(t_d,"pest_forgive.pst")) - m_d = os.path.join(model_d,"master_sweep_bin_base") - pyemu.os_utils.start_workers(t_d, exe_path, "pest_forgive.pst", 10, master_dir=m_d, - worker_root=model_d,port=port) - df1 = pd.read_csv(os.path.join(m_d, "pest_forgive.0.obs.csv"),index_col=0) - assert df1.shape[0] == pe.shape[0] - m_d = os.path.join(model_d, "master_sweep_bin") - pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-swp"), "pest_forgive.pst", 10, master_dir=m_d, - worker_root=model_d, port=port) - df2 = pyemu.Matrix.from_binary(os.path.join(m_d,"sweep_out.bin")).to_dataframe() - print(df2) - print(df1) - assert df2.shape == df1.shape - diff = (df1.values - df2.values) - print(diff) - print(diff.max()) - print(np.abs(diff).max()) - assert np.abs(diff).max() < 1e-7 - - +# pyemu.os_utils.run("mf6",cwd=new_d) +# pst = pyemu.Pst(os.path.join(base_d,"freyberg6_run_ies.pst")) +# pst.control_data.noptmax = 0 +# pst.pestpp_options = {} +# pst.pestpp_options["ies_par_en"] = "prior.jcb" + +# pst.write(os.path.join(new_d,"freyberg6_run_ies.pst"),version=2) +# pyemu.os_utils.run("pestpp-ies freyberg6_run_ies.pst",cwd=new_d) + +# build_and_draw_prior(new_d,num_reals=5000) +# pst.control_data.noptmax = -1 +# pst.write(os.path.join(new_d,"freyberg6_run_ies.pst"),version=2) +# m_d = os.path.join(model_d,"ends_master") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) + +# pyemu.os_utils.start_workers(new_d,"pestpp-ies","freyberg6_run_ies.pst",num_workers=15,worker_root=model_d,master_dir=m_d) + + +# def build_and_draw_prior(t_d="ends",num_reals=500): +# import flopy + +# sim = flopy.mf6.MFSimulation.load(sim_ws=t_d) +# m = sim.get_model("freyberg6") +# xgrid = m.modelgrid.xcellcenters +# ygrid = m.modelgrid.ycellcenters +# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst")) +# par = pst.parameter_data +# static_par = par.loc[par.parnme.apply(lambda x: x[:3] in ["npf","sto"]),:].copy() +# static_par.loc[:, "i"] = static_par.parnme.apply(lambda x: int(x.split('_')[3])) +# static_par.loc[:, "j"] = static_par.parnme.apply(lambda x: int(x.split('_')[4])) +# static_par.loc[:, "x"] = static_par.apply(lambda x: xgrid[x.i,x.j],axis=1) +# static_par.loc[:, "y"] = static_par.apply(lambda x: ygrid[x.i, x.j], axis=1) +# static_par.loc[:,"pargp"] = static_par.parnme.apply(lambda x: "_".join(x.split('_')[:3])) + +# wel_par = par.loc[par.parnme.apply(lambda x: x.startswith("wel")),:].copy() +# wel_par.loc[:,"x"] = wel_par.parnme.apply(lambda x: int(x.split('_')[-1])) +# wel_par.loc[:,"y"] = 0.0 +# wel_par.loc[:,"pargp"] = wel_par.parnme.apply(lambda x: '_'.join(x.split('_')[:-1])) + +# rch_par = par.loc[par.parnme.str.startswith("rch"),:].copy() +# rch_par.loc[:,"x"] = rch_par.parnme.apply(lambda x: int(x.split('_')[-1])) +# rch_par.loc[:,"y"] = 0.0 + +# spatial_v = pyemu.geostats.ExpVario(contribution=1.0,a=1000.0) +# temporal_v = pyemu.geostats.ExpVario(contribution=1.0,a=3) +# spatial_gs = pyemu.geostats.GeoStruct(variograms=spatial_v) +# temporal_gs = pyemu.geostats.GeoStruct(variograms=temporal_v) + +# static_struct_dict = {spatial_gs:[]} +# sgrps = static_par.pargp.unique() +# sgrps.sort() +# for pargp in sgrps: +# static_struct_dict[spatial_gs].append(static_par.loc[static_par.pargp==pargp,["parnme","x","y","i","j"]]) +# temporal_struct_dict = {temporal_gs: [rch_par.loc[:, ["parnme", "x", "y"]]]} +# wgrps = wel_par.pargp.unique() +# wgrps.sort() +# for pargp in wgrps: +# temporal_struct_dict[temporal_gs].append(wel_par.loc[wel_par.pargp == pargp, ["parnme", "x", "y"]]) + +# struct_dict = static_struct_dict +# for k,v in temporal_struct_dict.items(): +# struct_dict[k] = v +# print(struct_dict) +# np.random.seed(pyemu.en.SEED) +# pe = pyemu.helpers.geostatistical_draws(pst,struct_dict=struct_dict,num_reals=num_reals) +# pe.to_binary(os.path.join(t_d,"prior.jcb")) + + +# def run(): +# model_d = "mf6_freyberg" +# t_d = os.path.join(model_d,"template") +# pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" +# pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, +# worker_root=model_d, port=4004) + +# def sweep_bin_test(): + +# model_d = "ies_10par_xsec" +# t_d = os.path.join(model_d,"template") +# m_d = os.path.join(model_d,"master_sweep_bin") +# if os.path.exists(m_d): +# shutil.rmtree(m_d) +# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) +# pe = pyemu.ParameterEnsemble.from_uniform_draw(pst,num_reals=50)#.loc[:,pst.par_names[:2]] + +# pe.to_csv(os.path.join(t_d,"sweep_in.csv")) +# pe._df.index = pe.index.map(str) +# print(pe.index) +# pe.to_dense(os.path.join(t_d,"sweep_in.bin")) +# pst.pestpp_options["ies_par_en"] = "sweep_in.csv" +# pst.pestpp_options["sweep_forgive"] = True +# pst.pestpp_options["sweep_parameter_file"] = "sweep_in.bin" +# pst.control_data.noptmax = -1 +# pst.pestpp_options.pop("ies_num_reals",None) +# pst.write(os.path.join(t_d,"pest_forgive.pst")) +# pst.pestpp_options["sweep_output_file"] = "sweep_out.bin" +# pst.pestpp_options["sweep_chunk"] = 9 +# pst.pestpp_options["ies_include_base"] = False +# pst.write(os.path.join(t_d,"pest_forgive.pst")) +# m_d = os.path.join(model_d,"master_sweep_bin_base") +# pyemu.os_utils.start_workers(t_d, exe_path, "pest_forgive.pst", 10, master_dir=m_d, +# worker_root=model_d,port=port) +# df1 = pd.read_csv(os.path.join(m_d, "pest_forgive.0.obs.csv"),index_col=0) +# assert df1.shape[0] == pe.shape[0] +# m_d = os.path.join(model_d, "master_sweep_bin") +# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-swp"), "pest_forgive.pst", 10, master_dir=m_d, +# worker_root=model_d, port=port) +# df2 = pyemu.Matrix.from_binary(os.path.join(m_d,"sweep_out.bin")).to_dataframe() +# print(df2) +# print(df1) +# assert df2.shape == df1.shape +# diff = (df1.values - df2.values) +# print(diff) +# print(diff.max()) +# print(np.abs(diff).max()) +# assert np.abs(diff).max() < 1e-7 + +def fail_test(): + raise Exception("fail please") if __name__ == "__main__": #run() - exit(1) + fail_test() #mf6_v5_ies_test() #prep_ends() - sweep_bin_test() + #sweep_bin_test() #mf6_v5_sen_test() #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-glm.exe"),os.path.join("..","bin","win","pestpp-glm.exe")) #shutil.copy2(os.path.join("..", "exe", "windows", "x64", "Debug", "pestpp-ies.exe"), From cc0d3d0a73b9ecd9c58f86bd67049b7bce8ca735 Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 20:22:15 -0600 Subject: [PATCH 14/27] quick fail basic tests --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff63abcc..9330b9d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,7 +174,9 @@ jobs: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: run test script ${{ env.test_script }} in ${{ env.test_dir }} - run: | + shell: bash -l {0} + run: | + ls -l cd ${{ env.test_dir }} nosetests -v ${{ env.test_script }} From f137906f6a12c184d0b8b705ef3b21f6a8060d1f Mon Sep 17 00:00:00 2001 From: jwhite Date: Mon, 29 Apr 2024 20:23:05 -0600 Subject: [PATCH 15/27] quick fail basic tests --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9330b9d2..efe6b0b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,7 @@ jobs: ls -l cd ${{ env.test_dir }} nosetests -v ${{ env.test_script }} + echo $? cd From a0878741d71ba645c356aceefd8df6da077e64bb Mon Sep 17 00:00:00 2001 From: jwhite Date: Tue, 30 Apr 2024 07:32:39 -0600 Subject: [PATCH 16/27] quick fail basic tests --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efe6b0b6..7198e923 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,6 +181,7 @@ jobs: cd ${{ env.test_dir }} nosetests -v ${{ env.test_script }} echo $? + exit 1 cd From e9bcbcadfd9039f09aa8f7ff0c55f9562c002f00 Mon Sep 17 00:00:00 2001 From: jwhite Date: Tue, 30 Apr 2024 07:46:33 -0600 Subject: [PATCH 17/27] more CI BS --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7198e923..c1a96cf5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,7 +181,7 @@ jobs: cd ${{ env.test_dir }} nosetests -v ${{ env.test_script }} echo $? - exit 1 + exit $? cd From ad165e28eb9774805b395ffd0b61384920d3f1e1 Mon Sep 17 00:00:00 2001 From: jwhite Date: Tue, 30 Apr 2024 08:16:01 -0600 Subject: [PATCH 18/27] more CI BS --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1a96cf5..f6ee4b80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,8 +181,12 @@ jobs: cd ${{ env.test_dir }} nosetests -v ${{ env.test_script }} echo $? - exit $? - cd + + if [$? == 0];then + exit 0 + else + exit 127 + fi From d5729776e2931c64573b4580324552fe348303e9 Mon Sep 17 00:00:00 2001 From: jwhite Date: Tue, 30 Apr 2024 08:19:38 -0600 Subject: [PATCH 19/27] more CI BS --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6ee4b80..04caff9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,9 +180,10 @@ jobs: ls -l cd ${{ env.test_dir }} nosetests -v ${{ env.test_script }} - echo $? + status = $? + echo status - if [$? == 0];then + if [status == 0];then exit 0 else exit 127 From c5b785091cd4066f2132472d07f8e24430bf2ccd Mon Sep 17 00:00:00 2001 From: jwhite Date: Tue, 30 Apr 2024 10:16:38 -0600 Subject: [PATCH 20/27] more CI BS --- .github/workflows/ci.yml | 9 +- benchmarks/basic_tests.py | 2837 ++++++++--------- src/libs/common/system_variables.cpp | 2 +- src/libs/common/utilities.cpp | 2 +- .../pestpp_common/EnsembleMethodUtils.cpp | 14 +- .../abstract_base/model_interface.cpp | 39 +- .../abstract_base/model_interface.h | 2 + .../run_managers/serial/RunManagerSerial.cpp | 3 +- .../run_managers/serial/RunManagerSerial.h | 3 +- src/programs/pestpp-ies/pestpp-ies.cpp | 3 +- 10 files changed, 1464 insertions(+), 1450 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04caff9c..b0719d46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,15 +180,14 @@ jobs: ls -l cd ${{ env.test_dir }} nosetests -v ${{ env.test_script }} - status = $? - echo status - - if [status == 0];then + status=$? + echo $status + + if [ $status -eq 0 ]; then exit 0 else exit 127 fi - diff --git a/benchmarks/basic_tests.py b/benchmarks/basic_tests.py index 5ed3d838..a173203d 100644 --- a/benchmarks/basic_tests.py +++ b/benchmarks/basic_tests.py @@ -1,1495 +1,1494 @@ -# import os -# import sys -# import shutil -# import platform -# import numpy as np -# import pandas as pd -# import platform -# import pyemu - -# bin_path = os.path.join("test_bin") -# if "linux" in platform.platform().lower(): -# bin_path = os.path.join(bin_path,"linux") -# elif "darwin" in platform.platform().lower() or "macos" in platform.platform().lower() : -# bin_path = os.path.join(bin_path,"mac") -# else: -# bin_path = os.path.join(bin_path,"win") - -# bin_path = os.path.abspath("test_bin") -# os.environ["PATH"] += os.pathsep + bin_path - - -# # bin_path = os.path.join("..","..","..","bin") -# # exe = "" -# # if "windows" in platform.platform().lower(): -# # exe = ".exe" -# # exe_path = os.path.join(bin_path, "pestpp-ies" + exe) - -# # case of either appveyor, travis or local -# if os.path.exists(os.path.join("pestpp","bin")): -# bin_path = os.path.join("..","..","pestpp","bin") -# else: -# bin_path = os.path.join("..","..","..","..","pestpp","bin") +import os +import sys +import shutil +import platform +import numpy as np +import pandas as pd +import platform +import pyemu + +bin_path = os.path.join("test_bin") +if "linux" in platform.platform().lower(): + bin_path = os.path.join(bin_path,"linux") +elif "darwin" in platform.platform().lower() or "macos" in platform.platform().lower() : + bin_path = os.path.join(bin_path,"mac") +else: + bin_path = os.path.join(bin_path,"win") + +bin_path = os.path.abspath("test_bin") +os.environ["PATH"] += os.pathsep + bin_path + + +# bin_path = os.path.join("..","..","..","bin") +# exe = "" +# if "windows" in platform.platform().lower(): +# exe = ".exe" +# exe_path = os.path.join(bin_path, "pestpp-ies" + exe) + +# case of either appveyor, travis or local +if os.path.exists(os.path.join("pestpp","bin")): + bin_path = os.path.join("..","..","pestpp","bin") +else: + bin_path = os.path.join("..","..","..","..","pestpp","bin") -# if "windows" in platform.platform().lower(): -# exe_path = os.path.join(bin_path, "win", "pestpp-ies.exe") -# elif "darwin" in platform.platform().lower() or "macos" in platform.platform().lower() : -# exe_path = os.path.join(bin_path, "mac", "pestpp-ies") -# else: -# exe_path = os.path.join(bin_path, "linux", "pestpp-ies") - -# noptmax = 4 -# num_reals = 20 -# port = 4021 - -# def basic_test(model_d="ies_10par_xsec"): -# pyemu.Ensemble.reseed() -# base_d = os.path.join(model_d, "template") -# new_d = os.path.join(model_d, "test_template") -# if os.path.exists(new_d): -# shutil.rmtree(new_d) -# shutil.copytree(base_d, new_d) -# print(platform.platform().lower()) -# pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) -# print(pst.model_command) +if "windows" in platform.platform().lower(): + exe_path = os.path.join(bin_path, "win", "pestpp-ies.exe") +elif "darwin" in platform.platform().lower() or "macos" in platform.platform().lower() : + exe_path = os.path.join(bin_path, "mac", "pestpp-ies") +else: + exe_path = os.path.join(bin_path, "linux", "pestpp-ies") + +noptmax = 4 +num_reals = 20 +port = 4021 + +def basic_test(model_d="ies_10par_xsec"): + pyemu.Ensemble.reseed() + base_d = os.path.join(model_d, "template") + new_d = os.path.join(model_d, "test_template") + if os.path.exists(new_d): + shutil.rmtree(new_d) + shutil.copytree(base_d, new_d) + print(platform.platform().lower()) + pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) + print(pst.model_command) -# # set first par as fixed -# #pst.parameter_data.loc[pst.par_names[0], "partrans"] = "fixed" - -# pst.observation_data.loc[pst.nnz_obs_names,"weight"] = 1.0 - -# # set noptmax -# pst.control_data.noptmax = noptmax - -# # wipe all pestpp options -# pst.pestpp_options = {} -# pst.pestpp_options["ies_num_reals"] = num_reals -# pst.pestpp_options["lambda_scale_fac"] = [0.5,0.75,1.0] -# pst.pestpp_options["ies_lambda_mults"] = 1.0 -# # write a generic 2D cov -# if os.path.exists(os.path.join(new_d,"prior.jcb")): -# cov = pyemu.Cov.from_binary(os.path.join(new_d,"prior.jcb")) -# #cov.to_ascii(os.path.join(new_d,"prior.cov")) -# elif os.path.exists(os.path.join(new_d, "prior.cov")): -# cov = pyemu.Cov.from_ascii(os.path.join(new_d, "prior.cov")) -# else: -# cov = pyemu.Cov.from_parameter_data(pst) -# cov = pyemu.Cov(cov.as_2d, names=cov.row_names) -# #cov.to_ascii(os.path.join(new_d, "prior.cov")) -# cov.to_binary(os.path.join(new_d, "prior.jcb")) - -# # draw some ensembles -# idx = [i for i in range(num_reals)] -# idx[-1] = "base" -# pe = pyemu.ParameterEnsemble.from_gaussian_draw(pst, cov=cov, num_reals=num_reals) -# pe.index = idx -# pe.to_csv(os.path.join(new_d, "par.csv")) -# pe.to_binary(os.path.join(new_d, "par.jcb")) -# pe.to_csv(os.path.join(new_d, "sweep_in.csv")) -# pe.loc[:, pst.adj_par_names].to_csv(os.path.join(new_d, "par_some.csv")) -# pe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_par.csv")) -# oe = pyemu.ObservationEnsemble.from_gaussian_draw(pst, num_reals=num_reals) -# oe.index = idx -# oe.to_csv(os.path.join(new_d, "obs.csv")) -# oe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_base_obs.csv")) -# oe.to_binary(os.path.join(new_d, "obs.jcb")) - -# pst.control_data.noptmax = 0 -# pst.write(os.path.join(new_d, "pest.pst")) -# pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d) -# df = pd.read_csv(os.path.join(new_d,"pest.phi.group.csv")) -# assert df.loc[0,"head"] == 0.5,df -# #return -# pst.control_data.noptmax = noptmax -# pst.write(os.path.join(new_d, "pest.pst")) + # set first par as fixed + #pst.parameter_data.loc[pst.par_names[0], "partrans"] = "fixed" + + pst.observation_data.loc[pst.nnz_obs_names,"weight"] = 1.0 + + # set noptmax + pst.control_data.noptmax = noptmax + + # wipe all pestpp options + pst.pestpp_options = {} + pst.pestpp_options["ies_num_reals"] = num_reals + pst.pestpp_options["lambda_scale_fac"] = [0.5,0.75,1.0] + pst.pestpp_options["ies_lambda_mults"] = 1.0 + # write a generic 2D cov + if os.path.exists(os.path.join(new_d,"prior.jcb")): + cov = pyemu.Cov.from_binary(os.path.join(new_d,"prior.jcb")) + #cov.to_ascii(os.path.join(new_d,"prior.cov")) + elif os.path.exists(os.path.join(new_d, "prior.cov")): + cov = pyemu.Cov.from_ascii(os.path.join(new_d, "prior.cov")) + else: + cov = pyemu.Cov.from_parameter_data(pst) + cov = pyemu.Cov(cov.as_2d, names=cov.row_names) + #cov.to_ascii(os.path.join(new_d, "prior.cov")) + cov.to_binary(os.path.join(new_d, "prior.jcb")) + + # draw some ensembles + idx = [i for i in range(num_reals)] + idx[-1] = "base" + pe = pyemu.ParameterEnsemble.from_gaussian_draw(pst, cov=cov, num_reals=num_reals) + pe.index = idx + pe.to_csv(os.path.join(new_d, "par.csv")) + pe.to_binary(os.path.join(new_d, "par.jcb")) + pe.to_csv(os.path.join(new_d, "sweep_in.csv")) + pe.loc[:, pst.adj_par_names].to_csv(os.path.join(new_d, "par_some.csv")) + pe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_par.csv")) + oe = pyemu.ObservationEnsemble.from_gaussian_draw(pst, num_reals=num_reals) + oe.index = idx + oe.to_csv(os.path.join(new_d, "obs.csv")) + oe.iloc[:-3, :].to_csv(os.path.join(new_d, "restart_failed_base_obs.csv")) + oe.to_binary(os.path.join(new_d, "obs.jcb")) + + pst.control_data.noptmax = 0 + pst.write(os.path.join(new_d, "pest.pst")) + pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d) + df = pd.read_csv(os.path.join(new_d,"pest.phi.group.csv")) + assert df.loc[0,"head"] == 0.5,df + #return + pst.control_data.noptmax = noptmax + pst.write(os.path.join(new_d, "pest.pst")) -# m_d = os.path.join(model_d,"master_pestpp_sen") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-sen"), "pest.pst", 5, master_dir=m_d, -# worker_root=model_d,port=port,verbose=True) -# #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-sen")),cwd=new_d) -# df = pd.read_csv(os.path.join(m_d, "pest.mio"),index_col=0) - -# # run sweep -# m_d = os.path.join(model_d,"master_sweep1") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-swp"), "pest.pst", 5, master_dir=m_d, -# worker_root=model_d,port=port,verbose=True) -# df = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) + m_d = os.path.join(model_d,"master_pestpp_sen") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-sen"), "pest.pst", 5, master_dir=m_d, + worker_root=model_d,port=port,verbose=True) + #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-sen")),cwd=new_d) + df = pd.read_csv(os.path.join(m_d, "pest.mio"),index_col=0) + + # run sweep + m_d = os.path.join(model_d,"master_sweep1") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-swp"), "pest.pst", 5, master_dir=m_d, + worker_root=model_d,port=port,verbose=True) + df = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) -# m_d = os.path.join(model_d,"master_pestpp-glm") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-glm"), "pest.pst", 10, master_dir=m_d, -# worker_root=model_d,port=port,verbose=True) -# #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=new_d) -# df = pd.read_csv(os.path.join(m_d, "pest.par.usum.csv"),index_col=0) - -# m_d = os.path.join(model_d,"master_pestpp-ies") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pyemu.os_utils.start_workers(new_d, exe_path, "pest.pst", 10, master_dir=m_d, -# worker_root=model_d,port=port,verbose=True) - - - -# def glm_save_binary_test(): -# model_d = "ies_10par_xsec" - -# t_d = os.path.join(model_d, "template") -# m_d = os.path.join(model_d, "master_save_binary") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d, "pest.pst")) -# pst.pestpp_options = {"glm_num_reals":30,"save_binary":True} -# pst.control_data.noptmax = 1 -# pst.write(os.path.join(t_d, "pest_save_binary.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-glm"), "pest_save_binary.pst", 10, master_dir=m_d, -# worker_root=model_d, port=port) - -# pe = pyemu.ParameterEnsemble.from_binary(pst=pst,filename=os.path.join(m_d,"pest_save_binary.post.paren.jcb")) -# pe = pyemu.ObservationEnsemble.from_binary(pst=pst,filename=os.path.join(m_d, "pest_save_binary.post.obsen.jcb")) - -# def sweep_forgive_test(): -# model_d = "ies_10par_xsec" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_sweep_forgive") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) -# pe = pyemu.ParameterEnsemble.from_uniform_draw(pst,num_reals=50)#.loc[:,pst.par_names[:2]] -# pe.loc[:,pst.par_names[2:]] = pst.parameter_data.loc[pst.par_names[2:],"parval1"].values -# pe.to_csv(os.path.join(t_d,"sweep_in.csv")) -# pst.pestpp_options["sweep_forgive"] = True -# pst.control_data.noptmax = -1 -# pst.pestpp_options["ies_num_reals"] = 5 -# pst.write(os.path.join(t_d,"pest_forgive.pst")) - - -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_forgive.pst", 10, master_dir=m_d, -# worker_root=model_d,port=port) -# df1 = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) - -# pe = pe.loc[:,pst.par_names[:2]] -# pe.to_csv(os.path.join(t_d,"sweep_in.csv")) -# pst.pestpp_options["sweep_forgive"] = True -# pst.write(os.path.join(t_d,"pest_forgive.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_forgive.pst", 10, master_dir=m_d, -# worker_root=model_d,port=port) -# df2 = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) -# diff = df1 - df2 -# print(diff.max()) -# assert diff.max().max() == 0.0 - - - - -# def inv_regul_test(): -# model_d = "ies_10par_xsec" + m_d = os.path.join(model_d,"master_pestpp-glm") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pyemu.os_utils.start_workers(new_d, exe_path.replace("-ies","-glm"), "pest.pst", 10, master_dir=m_d, + worker_root=model_d,port=port,verbose=True) + #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=new_d) + df = pd.read_csv(os.path.join(m_d, "pest.par.usum.csv"),index_col=0) + + m_d = os.path.join(model_d,"master_pestpp-ies") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pyemu.os_utils.start_workers(new_d, exe_path, "pest.pst", 10, master_dir=m_d, + worker_root=model_d,port=port,verbose=True) + + + +def glm_save_binary_test(): + model_d = "ies_10par_xsec" + + t_d = os.path.join(model_d, "template") + m_d = os.path.join(model_d, "master_save_binary") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d, "pest.pst")) + pst.pestpp_options = {"glm_num_reals":30,"save_binary":True} + pst.control_data.noptmax = 1 + pst.write(os.path.join(t_d, "pest_save_binary.pst")) + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-glm"), "pest_save_binary.pst", 10, master_dir=m_d, + worker_root=model_d, port=port) + + pe = pyemu.ParameterEnsemble.from_binary(pst=pst,filename=os.path.join(m_d,"pest_save_binary.post.paren.jcb")) + pe = pyemu.ObservationEnsemble.from_binary(pst=pst,filename=os.path.join(m_d, "pest_save_binary.post.obsen.jcb")) + +def sweep_forgive_test(): + model_d = "ies_10par_xsec" + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_sweep_forgive") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) + pe = pyemu.ParameterEnsemble.from_uniform_draw(pst,num_reals=50)#.loc[:,pst.par_names[:2]] + pe.loc[:,pst.par_names[2:]] = pst.parameter_data.loc[pst.par_names[2:],"parval1"].values + pe.to_csv(os.path.join(t_d,"sweep_in.csv")) + pst.pestpp_options["sweep_forgive"] = True + pst.control_data.noptmax = -1 + pst.pestpp_options["ies_num_reals"] = 5 + pst.write(os.path.join(t_d,"pest_forgive.pst")) + + + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_forgive.pst", 10, master_dir=m_d, + worker_root=model_d,port=port) + df1 = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) + + pe = pe.loc[:,pst.par_names[:2]] + pe.to_csv(os.path.join(t_d,"sweep_in.csv")) + pst.pestpp_options["sweep_forgive"] = True + pst.write(os.path.join(t_d,"pest_forgive.pst")) + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_forgive.pst", 10, master_dir=m_d, + worker_root=model_d,port=port) + df2 = pd.read_csv(os.path.join(m_d, "sweep_out.csv"),index_col=0) + diff = df1 - df2 + print(diff.max()) + assert diff.max().max() == 0.0 + + + + +def inv_regul_test(): + model_d = "ies_10par_xsec" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_inv_regul") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) -# #pyemu.helpers.zero_order_tikhonov(pst) -# #pst.control_data.pestmode = "regularization" -# pst.reg_data.phimlim = 2 -# pst.reg_data.phimaccept = 2.2 -# pst.control_data.noptmax = 10 -# pst.write(os.path.join(t_d,"pest_regul.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_regul.pst", 10, master_dir=m_d, -# worker_root=model_d,port=port) + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_inv_regul") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) + #pyemu.helpers.zero_order_tikhonov(pst) + #pst.control_data.pestmode = "regularization" + pst.reg_data.phimlim = 2 + pst.reg_data.phimaccept = 2.2 + pst.control_data.noptmax = 10 + pst.write(os.path.join(t_d,"pest_regul.pst")) + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_regul.pst", 10, master_dir=m_d, + worker_root=model_d,port=port) -# def tie_by_group_test(): -# model_d = "ies_10par_xsec" +def tie_by_group_test(): + model_d = "ies_10par_xsec" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_tie_by_group") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) -# par = pst.parameter_data -# tied_names = pst.adj_par_names[:3] -# par.loc[tied_names[1:3],"partrans"] = "tied" -# par.loc[tied_names[1:3],"partied"] = tied_names[0] -# par.loc[tied_names[1:3],"parval1"] = 1.0 -# par.loc[tied_names[1:3],"parubnd"] = par.loc[tied_names[1:3],"parval1"] * 1.0001 -# par.loc[tied_names[1:3],"parlbnd"] = par.loc[tied_names[1:3],"parval1"] * 0.9999 -# pst.pestpp_options = {} -# pst.pestpp_options["ies_num_reals"] = 10 -# pst.pestpp_options["ies_lambda_mults"] = 1.0 -# pst.pestpp_options["lambda_scale_fac"] = 1.0 -# pst.pestpp_options["tie_by_group"] = True -# pst.pestpp_options["ies_include_base"] = False -# pst.pestpp_options["enforce_tied_bounds"] = True -# pst.control_data.noptmax = 1 - - -# pst.write(os.path.join(t_d,"pest_tied.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "pest_tied.pst", 5, master_dir=m_d, -# worker_root=model_d,port=port) -# df = pd.read_csv(os.path.join(m_d,"pest_tied.sen.par.csv"),index_col=0) -# df.columns = df.columns.str.lower() -# print(df.loc[:,tied_names[1:3]]) -# print(df.loc[:,tied_names[1:3]].std(axis=1)) -# print(df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max()) -# assert df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max() < 1.0e-8 -# for real in df.index: -# too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] -# assert too_low.shape[0] == 0, "sen,{0},{1}".format(real,too_low) -# too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] -# assert too_high.shape[0] == 0, "sen,{0},{1}".format(real,too_high) + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_tie_by_group") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) + par = pst.parameter_data + tied_names = pst.adj_par_names[:3] + par.loc[tied_names[1:3],"partrans"] = "tied" + par.loc[tied_names[1:3],"partied"] = tied_names[0] + par.loc[tied_names[1:3],"parval1"] = 1.0 + par.loc[tied_names[1:3],"parubnd"] = par.loc[tied_names[1:3],"parval1"] * 1.0001 + par.loc[tied_names[1:3],"parlbnd"] = par.loc[tied_names[1:3],"parval1"] * 0.9999 + pst.pestpp_options = {} + pst.pestpp_options["ies_num_reals"] = 10 + pst.pestpp_options["ies_lambda_mults"] = 1.0 + pst.pestpp_options["lambda_scale_fac"] = 1.0 + pst.pestpp_options["tie_by_group"] = True + pst.pestpp_options["ies_include_base"] = False + pst.pestpp_options["enforce_tied_bounds"] = True + pst.control_data.noptmax = 1 + + + pst.write(os.path.join(t_d,"pest_tied.pst")) + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "pest_tied.pst", 5, master_dir=m_d, + worker_root=model_d,port=port) + df = pd.read_csv(os.path.join(m_d,"pest_tied.sen.par.csv"),index_col=0) + df.columns = df.columns.str.lower() + print(df.loc[:,tied_names[1:3]]) + print(df.loc[:,tied_names[1:3]].std(axis=1)) + print(df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max()) + assert df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max() < 1.0e-8 + for real in df.index: + too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] + assert too_low.shape[0] == 0, "sen,{0},{1}".format(real,too_low) + too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] + assert too_high.shape[0] == 0, "sen,{0},{1}".format(real,too_high) -# #pst.write(os.path.join(t_d,"pest_tied.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_tied.pst", 5, master_dir=m_d, -# worker_root=model_d,port=port) -# jco = pyemu.Jco.from_binary(os.path.join(m_d,"pest_tied.jcb")) -# assert jco.shape[1] == 2,jco.shape -# par_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_tied.par")) -# print(par_df) -# too_low = par.loc[par_df.parval1 < par.parlbnd,"parnme"] -# assert too_low.shape[0] == 0,too_low -# too_high = par.loc[par_df.parval1 > par.parubnd, "parnme"] -# assert too_high.shape[0] == 0, too_high + #pst.write(os.path.join(t_d,"pest_tied.pst")) + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_tied.pst", 5, master_dir=m_d, + worker_root=model_d,port=port) + jco = pyemu.Jco.from_binary(os.path.join(m_d,"pest_tied.jcb")) + assert jco.shape[1] == 2,jco.shape + par_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_tied.par")) + print(par_df) + too_low = par.loc[par_df.parval1 < par.parlbnd,"parnme"] + assert too_low.shape[0] == 0,too_low + too_high = par.loc[par_df.parval1 > par.parubnd, "parnme"] + assert too_high.shape[0] == 0, too_high -# pst.control_data.noptmax = 1 -# pst.write(os.path.join(t_d, "pest_tied.pst")) + pst.control_data.noptmax = 1 + pst.write(os.path.join(t_d, "pest_tied.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path, "pest_tied.pst", 10, master_dir=m_d, -# worker_root=model_d,port=port) + pyemu.os_utils.start_workers(t_d, exe_path, "pest_tied.pst", 10, master_dir=m_d, + worker_root=model_d,port=port) -# for nopt in range(pst.control_data.noptmax+1): -# df = pd.read_csv(os.path.join(m_d,"pest_tied.{0}.par.csv".format(nopt)),index_col=0) -# df.columns = df.columns.str.lower() -# print(df.loc[:,tied_names[1:3]]) -# print(df.loc[:,tied_names[1:3]].std(axis=1)) -# print(df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max()) -# assert df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max() < 1.0e-8 -# for real in df.index: -# too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] -# assert too_low.shape[0] == 0, "ies,{0},{1},{2}".format(nopt,real,too_low) -# too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] -# assert too_high.shape[0] == 0, "ies,{0},{1},{2}".format(nopt,real,too_high) - - -# par.loc[tied_names[1:3],"parval1"] = par.loc[tied_names[0],"parval1"] -# print(par.parval1) -# par.loc[tied_names[1:3], "parubnd"] = par.loc[tied_names[1:3], "parval1"] * 1.5 -# par.loc[tied_names[1:3], "parlbnd"] = par.loc[tied_names[1:3], "parval1"] * 0.5 -# pst.pestpp_options["ies_num_reals"] = 10 -# pst.pestpp_options["ies_lambda_mults"] = 1.0 -# pst.pestpp_options["lambda_scale_fac"] = 1.0 -# pst.pestpp_options["tie_by_group"] = True - - -# pst.control_data.noptmax = 3 -# pst.write(os.path.join(t_d, "pest_tied.pst")) - -# pyemu.os_utils.start_workers(t_d, exe_path, "pest_tied.pst", 10, master_dir=m_d, -# worker_root=model_d, port=port) -# df = pd.read_csv(os.path.join(m_d, "pest_tied.{0}.par.csv".format(pst.control_data.noptmax)), index_col=0) -# df.columns = df.columns.str.lower() -# print(df.loc[:, tied_names].std(axis=1).apply(np.abs).max()) -# assert df.loc[:, tied_names].std(axis=1).apply(np.abs).max() < 1.0e-8 -# for real in df.index: -# too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] -# assert too_low.shape[0] == 0, "ies,{0},{1}".format(real,too_low) -# too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] -# assert too_high.shape[0] == 0, "ies,{0},{1}".format(real,too_high) + for nopt in range(pst.control_data.noptmax+1): + df = pd.read_csv(os.path.join(m_d,"pest_tied.{0}.par.csv".format(nopt)),index_col=0) + df.columns = df.columns.str.lower() + print(df.loc[:,tied_names[1:3]]) + print(df.loc[:,tied_names[1:3]].std(axis=1)) + print(df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max()) + assert df.loc[:,tied_names[1:3]].std(axis=1).apply(np.abs).max() < 1.0e-8 + for real in df.index: + too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] + assert too_low.shape[0] == 0, "ies,{0},{1},{2}".format(nopt,real,too_low) + too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] + assert too_high.shape[0] == 0, "ies,{0},{1},{2}".format(nopt,real,too_high) + + + par.loc[tied_names[1:3],"parval1"] = par.loc[tied_names[0],"parval1"] + print(par.parval1) + par.loc[tied_names[1:3], "parubnd"] = par.loc[tied_names[1:3], "parval1"] * 1.5 + par.loc[tied_names[1:3], "parlbnd"] = par.loc[tied_names[1:3], "parval1"] * 0.5 + pst.pestpp_options["ies_num_reals"] = 10 + pst.pestpp_options["ies_lambda_mults"] = 1.0 + pst.pestpp_options["lambda_scale_fac"] = 1.0 + pst.pestpp_options["tie_by_group"] = True + + + pst.control_data.noptmax = 3 + pst.write(os.path.join(t_d, "pest_tied.pst")) + + pyemu.os_utils.start_workers(t_d, exe_path, "pest_tied.pst", 10, master_dir=m_d, + worker_root=model_d, port=port) + df = pd.read_csv(os.path.join(m_d, "pest_tied.{0}.par.csv".format(pst.control_data.noptmax)), index_col=0) + df.columns = df.columns.str.lower() + print(df.loc[:, tied_names].std(axis=1).apply(np.abs).max()) + assert df.loc[:, tied_names].std(axis=1).apply(np.abs).max() < 1.0e-8 + for real in df.index: + too_low = df.loc[real,df.loc[real,par.parnme] < par.parlbnd] + assert too_low.shape[0] == 0, "ies,{0},{1}".format(real,too_low) + too_high = df.loc[real, df.loc[real, par.parnme] > par.parubnd] + assert too_high.shape[0] == 0, "ies,{0},{1}".format(real,too_high) -# df.to_csv(os.path.join(t_d,"sweep_in.csv")) -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_tied.pst", 5, master_dir=m_d, -# worker_root=model_d,port=port) -# pst.control_data.noptmax = 3 -# pst.pestpp_options["enforce_tied_bounds"] = False -# pst.write(os.path.join(t_d, "pest_tied.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_tied.pst", 5, master_dir=m_d, -# worker_root=model_d,port=port) -# jco = pyemu.Jco.from_binary(os.path.join(m_d,"pest_tied.jcb")) -# assert jco.shape[1] == 2,jco.shape + df.to_csv(os.path.join(t_d,"sweep_in.csv")) + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-swp"), "pest_tied.pst", 5, master_dir=m_d, + worker_root=model_d,port=port) + pst.control_data.noptmax = 3 + pst.pestpp_options["enforce_tied_bounds"] = False + pst.write(os.path.join(t_d, "pest_tied.pst")) + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-glm"), "pest_tied.pst", 5, master_dir=m_d, + worker_root=model_d,port=port) + jco = pyemu.Jco.from_binary(os.path.join(m_d,"pest_tied.jcb")) + assert jco.shape[1] == 2,jco.shape -# def unc_file_test(): -# model_d = "ies_10par_xsec" +def unc_file_test(): + model_d = "ies_10par_xsec" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_uncfile") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# shutil.copytree(t_d,m_d) -# pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) -# cov = pyemu.Cov.from_parameter_data(pst) -# cov.to_uncfile(os.path.join(m_d,"pest.unc"),covmat_file=os.path.join(m_d,"cov.mat"),var_mult=2.0,include_path=False) -# pst.pestpp_options = {} -# pst.pestpp_options["parcov"] = "pest.unc" -# pst.pestpp_options["ies_num_reals"] = 10000 -# pst.pestpp_options["ies_verbose_level"] = 3 -# pst.control_data.noptmax = -2 -# pst.write(os.path.join(m_d,"pest_unc.pst")) -# pyemu.os_utils.run("{0} {1}".format(exe_path,"pest_unc.pst"),cwd=m_d) - -# pe_1 = pd.read_csv(os.path.join(m_d,"pest_unc.0.par.csv"),index_col=0).apply(np.log10) - -# cov = pyemu.Cov.from_parameter_data(pst) -# cov *= 2.0 -# cov.to_uncfile(os.path.join(m_d,"pest.unc"),covmat_file=os.path.join(m_d,"cov.mat"),var_mult=1.0,include_path=False) -# pst.pestpp_options = {} -# pst.pestpp_options["parcov"] = "cov.mat" -# pst.pestpp_options["ies_num_reals"] = 10000 -# pst.pestpp_options["ies_verbose_level"] = 3 -# pst.control_data.noptmax = -2 -# pst.write(os.path.join(m_d,"pest_unc.pst")) -# pyemu.os_utils.run("{0} {1}".format(exe_path,"pest_unc.pst"),cwd=m_d) -# pe_2 = pd.read_csv(os.path.join(m_d,"pest_unc.0.par.csv"),index_col=0).apply(np.log10) -# diff = pe_1 - pe_2 -# print(pe_1.std(ddof=0)**2) -# print(pe_2.std(ddof=0)**2) -# print(diff.sum()) -# assert diff.sum().max() < 1.0e-10 - -# cov.to_uncfile(os.path.join(m_d, "pest.unc"), covmat_file=None) -# pst.control_data.noptmax = -2 -# pst.pestpp_options["ies_num_reals"] = 100000 -# pst.pestpp_options["ies_enforce_bounds"] = False -# pst.write(os.path.join(m_d, "pest_unc.pst")) -# pyemu.os_utils.run("{0} {1}".format(exe_path, "pest_unc.pst"), cwd=m_d) -# pe_3 = pd.read_csv(os.path.join(m_d, "pest_unc.0.par.csv"), index_col=0).apply(np.log10) -# print(pe_3.std(ddof=0)) -# pe_std = pe_3.std(ddof=0) -# for r,v in zip(cov.row_names,cov.x): -# d = np.abs(pe_std.loc[r] - np.sqrt(v)) - -# print(r,v,np.sqrt(v),d) -# assert d < 0.01 -# pst.control_data.noptmax = -1 -# pst.write(os.path.join(m_d, "pest_unc.pst")) -# pyemu.os_utils.run("{0} {1}".format(exe_path.replace("-ies","-glm"), "pest_unc.pst"), cwd=m_d) -# fosm_df = pd.read_csv(os.path.join(m_d,"pest_unc.par.usum.csv"),index_col=0) -# cov_df = cov.to_dataframe() -# for pname,prior_std in zip(fosm_df.index,fosm_df.prior_stdev): -# d = np.abs(prior_std - np.sqrt(cov_df.loc[pname,pname])) -# print(pname,d) -# assert d < 1.0e-4 - - -# def parchglim_test(): -# model_d = "ies_10par_xsec" + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_uncfile") + if os.path.exists(m_d): + shutil.rmtree(m_d) + shutil.copytree(t_d,m_d) + pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) + cov = pyemu.Cov.from_parameter_data(pst) + cov.to_uncfile(os.path.join(m_d,"pest.unc"),covmat_file=os.path.join(m_d,"cov.mat"),var_mult=2.0,include_path=False) + pst.pestpp_options = {} + pst.pestpp_options["parcov"] = "pest.unc" + pst.pestpp_options["ies_num_reals"] = 10000 + pst.pestpp_options["ies_verbose_level"] = 3 + pst.control_data.noptmax = -2 + pst.write(os.path.join(m_d,"pest_unc.pst")) + pyemu.os_utils.run("{0} {1}".format(exe_path,"pest_unc.pst"),cwd=m_d) + + pe_1 = pd.read_csv(os.path.join(m_d,"pest_unc.0.par.csv"),index_col=0).apply(np.log10) + + cov = pyemu.Cov.from_parameter_data(pst) + cov *= 2.0 + cov.to_uncfile(os.path.join(m_d,"pest.unc"),covmat_file=os.path.join(m_d,"cov.mat"),var_mult=1.0,include_path=False) + pst.pestpp_options = {} + pst.pestpp_options["parcov"] = "cov.mat" + pst.pestpp_options["ies_num_reals"] = 10000 + pst.pestpp_options["ies_verbose_level"] = 3 + pst.control_data.noptmax = -2 + pst.write(os.path.join(m_d,"pest_unc.pst")) + pyemu.os_utils.run("{0} {1}".format(exe_path,"pest_unc.pst"),cwd=m_d) + pe_2 = pd.read_csv(os.path.join(m_d,"pest_unc.0.par.csv"),index_col=0).apply(np.log10) + diff = pe_1 - pe_2 + print(pe_1.std(ddof=0)**2) + print(pe_2.std(ddof=0)**2) + print(diff.sum()) + assert diff.sum().max() < 1.0e-10 + + cov.to_uncfile(os.path.join(m_d, "pest.unc"), covmat_file=None) + pst.control_data.noptmax = -2 + pst.pestpp_options["ies_num_reals"] = 100000 + pst.pestpp_options["ies_enforce_bounds"] = False + pst.write(os.path.join(m_d, "pest_unc.pst")) + pyemu.os_utils.run("{0} {1}".format(exe_path, "pest_unc.pst"), cwd=m_d) + pe_3 = pd.read_csv(os.path.join(m_d, "pest_unc.0.par.csv"), index_col=0).apply(np.log10) + print(pe_3.std(ddof=0)) + pe_std = pe_3.std(ddof=0) + for r,v in zip(cov.row_names,cov.x): + d = np.abs(pe_std.loc[r] - np.sqrt(v)) + + print(r,v,np.sqrt(v),d) + assert d < 0.01 + pst.control_data.noptmax = -1 + pst.write(os.path.join(m_d, "pest_unc.pst")) + pyemu.os_utils.run("{0} {1}".format(exe_path.replace("-ies","-glm"), "pest_unc.pst"), cwd=m_d) + fosm_df = pd.read_csv(os.path.join(m_d,"pest_unc.par.usum.csv"),index_col=0) + cov_df = cov.to_dataframe() + for pname,prior_std in zip(fosm_df.index,fosm_df.prior_stdev): + d = np.abs(prior_std - np.sqrt(cov_df.loc[pname,pname])) + print(pname,d) + assert d < 1.0e-4 + + +def parchglim_test(): + model_d = "ies_10par_xsec" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_parchglim") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# shutil.copytree(t_d,m_d) -# pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) -# fpm = 1.05 -# pst.control_data.facparmax = fpm -# par = pst.parameter_data -# par.loc[pst.par_names[1:],"partrans"] = "fixed" -# par.loc[pst.par_names[0],"partrans"] = "log" -# par.loc[pst.par_names[0],"parchglim"] = "factor" -# par.loc[pst.par_names[0],"parval1"] = 1.0 + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_parchglim") + if os.path.exists(m_d): + shutil.rmtree(m_d) + shutil.copytree(t_d,m_d) + pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) + fpm = 1.05 + pst.control_data.facparmax = fpm + par = pst.parameter_data + par.loc[pst.par_names[1:],"partrans"] = "fixed" + par.loc[pst.par_names[0],"partrans"] = "log" + par.loc[pst.par_names[0],"parchglim"] = "factor" + par.loc[pst.par_names[0],"parval1"] = 1.0 -# pst.control_data.noptmax = 1 -# pst.pestpp_options["lambdas"] = 1.0 -# pst.pestpp_options["lambda_scale_fac"] = 1.0 -# pst.write(os.path.join(m_d,"pest_parchglim.pst")) -# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) -# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) -# print(p_df.loc["stage","parval1"],fpm) -# assert p_df.loc["stage","parval1"] == fpm - -# rpm = 0.1 -# par.loc[pst.par_names[0],"parchglim"] = "relative" -# pst.control_data.relparmax = rpm -# pst.write(os.path.join(m_d,"pest_parchglim.pst")) -# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) -# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) -# print(par) -# print(p_df) -# assert p_df.loc["stage","parval1"] == par.loc["stage","parval1"] + (rpm * par.loc["stage","parval1"]) - - -# par.loc[pst.par_names[0],"partrans"] = "none" -# par.loc[pst.par_names[0],"parlbnd"] = -10.0 -# par.loc[pst.par_names[0],"parubnd"] = 0.0 -# par.loc[pst.par_names[0],"parchglim"] = "factor" -# par.loc[pst.par_names[0],"parval1"] = -1.0 -# pst.write(os.path.join(m_d,"pest_parchglim.pst")) -# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) -# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) -# print(p_df) -# print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + np.abs(par.loc["stage","parval1"] * (fpm-1))) -# assert p_df.loc["stage","parval1"] <= par.loc["stage","parval1"] + np.abs(par.loc["stage","parval1"] * (fpm-1)) - -# rpm = 1.1 -# par.loc[pst.par_names[0],"partrans"] = "none" -# par.loc[pst.par_names[0],"parlbnd"] = -10.0 -# par.loc[pst.par_names[0],"parubnd"] = 10.0 -# par.loc[pst.par_names[0],"parchglim"] = "relative" -# par.loc[pst.par_names[0],"parval1"] = -1.0 -# pst.control_data.relparmax = rpm -# pst.write(os.path.join(m_d,"pest_parchglim.pst")) -# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) -# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) -# print(p_df) -# print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + rpm) -# assert np.abs(p_df.loc["stage","parval1"] - (par.loc["stage","parval1"] + rpm)) < 1.0e-6 - - -# par.loc[pst.par_names[1:],"partrans"] = "log" -# par.loc[pst.par_names[1:],"parchglim"] = "factor" -# pst.control_data.facparmax = 5.0 + pst.control_data.noptmax = 1 + pst.pestpp_options["lambdas"] = 1.0 + pst.pestpp_options["lambda_scale_fac"] = 1.0 + pst.write(os.path.join(m_d,"pest_parchglim.pst")) + pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) + p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) + print(p_df.loc["stage","parval1"],fpm) + assert p_df.loc["stage","parval1"] == fpm + + rpm = 0.1 + par.loc[pst.par_names[0],"parchglim"] = "relative" + pst.control_data.relparmax = rpm + pst.write(os.path.join(m_d,"pest_parchglim.pst")) + pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) + p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) + print(par) + print(p_df) + assert p_df.loc["stage","parval1"] == par.loc["stage","parval1"] + (rpm * par.loc["stage","parval1"]) + + + par.loc[pst.par_names[0],"partrans"] = "none" + par.loc[pst.par_names[0],"parlbnd"] = -10.0 + par.loc[pst.par_names[0],"parubnd"] = 0.0 + par.loc[pst.par_names[0],"parchglim"] = "factor" + par.loc[pst.par_names[0],"parval1"] = -1.0 + pst.write(os.path.join(m_d,"pest_parchglim.pst")) + pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) + p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) + print(p_df) + print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + np.abs(par.loc["stage","parval1"] * (fpm-1))) + assert p_df.loc["stage","parval1"] <= par.loc["stage","parval1"] + np.abs(par.loc["stage","parval1"] * (fpm-1)) + + rpm = 1.1 + par.loc[pst.par_names[0],"partrans"] = "none" + par.loc[pst.par_names[0],"parlbnd"] = -10.0 + par.loc[pst.par_names[0],"parubnd"] = 10.0 + par.loc[pst.par_names[0],"parchglim"] = "relative" + par.loc[pst.par_names[0],"parval1"] = -1.0 + pst.control_data.relparmax = rpm + pst.write(os.path.join(m_d,"pest_parchglim.pst")) + pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) + p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) + print(p_df) + print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + rpm) + assert np.abs(p_df.loc["stage","parval1"] - (par.loc["stage","parval1"] + rpm)) < 1.0e-6 + + + par.loc[pst.par_names[1:],"partrans"] = "log" + par.loc[pst.par_names[1:],"parchglim"] = "factor" + pst.control_data.facparmax = 5.0 -# pst.write(os.path.join(m_d,"pest_parchglim.pst")) -# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) -# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) -# print(p_df) -# print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + rpm) -# d = np.abs(p_df.loc["stage","parval1"] - (par.loc["stage","parval1"] + rpm)) -# assert d < 1.0e-6,d - -# rpm = 1.1 -# par.loc[pst.par_names[1:],"partrans"] = "fixed" -# par.loc[pst.par_names[1:],"parchglim"] = "factor" -# par.loc[pst.par_names[0],"partrans"] = "none" -# par.loc[pst.par_names[0],"parlbnd"] = -10.0 -# par.loc[pst.par_names[0],"parubnd"] = 10.0 -# par.loc[pst.par_names[0],"parchglim"] = "relative" -# par.loc[pst.par_names[0],"parval1"] = 0.0 -# pst.control_data.relparmax = rpm -# pst.write(os.path.join(m_d,"pest_parchglim.pst")) -# try: - -# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) -# except: -# pass -# else: -# raise Exception("should have failed") + pst.write(os.path.join(m_d,"pest_parchglim.pst")) + pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) + p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) + print(p_df) + print(p_df.loc["stage","parval1"],par.loc["stage","parval1"] + rpm) + d = np.abs(p_df.loc["stage","parval1"] - (par.loc["stage","parval1"] + rpm)) + assert d < 1.0e-6,d + + rpm = 1.1 + par.loc[pst.par_names[1:],"partrans"] = "fixed" + par.loc[pst.par_names[1:],"parchglim"] = "factor" + par.loc[pst.par_names[0],"partrans"] = "none" + par.loc[pst.par_names[0],"parlbnd"] = -10.0 + par.loc[pst.par_names[0],"parubnd"] = 10.0 + par.loc[pst.par_names[0],"parchglim"] = "relative" + par.loc[pst.par_names[0],"parval1"] = 0.0 + pst.control_data.relparmax = rpm + pst.write(os.path.join(m_d,"pest_parchglim.pst")) + try: + + pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) + except: + pass + else: + raise Exception("should have failed") -# rpm = 100 -# fpm = 100 -# par.loc[pst.par_names[1:],"partrans"] = "fixed" -# par.loc[pst.par_names[1:],"parchglim"] = "factor" -# par.loc[pst.par_names[0],"partrans"] = "none" -# par.loc[pst.par_names[0],"parlbnd"] = 0.9 -# par.loc[pst.par_names[0],"parubnd"] = 1.1 -# par.loc[pst.par_names[0],"parchglim"] = "relative" -# par.loc[pst.par_names[0],"parval1"] = 1.0 -# pst.control_data.relparmax = rpm -# pst.control_data.facparmax = fpm + rpm = 100 + fpm = 100 + par.loc[pst.par_names[1:],"partrans"] = "fixed" + par.loc[pst.par_names[1:],"parchglim"] = "factor" + par.loc[pst.par_names[0],"partrans"] = "none" + par.loc[pst.par_names[0],"parlbnd"] = 0.9 + par.loc[pst.par_names[0],"parubnd"] = 1.1 + par.loc[pst.par_names[0],"parchglim"] = "relative" + par.loc[pst.par_names[0],"parval1"] = 1.0 + pst.control_data.relparmax = rpm + pst.control_data.facparmax = fpm -# pst.write(os.path.join(m_d,"pest_parchglim.pst")) -# pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) -# p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) -# print(p_df) -# assert p_df.loc["stage","parval1"] == par.loc["stage","parubnd"] + pst.write(os.path.join(m_d,"pest_parchglim.pst")) + pyemu.os_utils.run("{0} pest_parchglim.pst".format(exe_path.replace("-ies","-glm")),cwd=m_d) + p_df = pyemu.pst_utils.read_parfile(os.path.join(m_d,"pest_parchglim.par")) + print(p_df) + assert p_df.loc["stage","parval1"] == par.loc["stage","parubnd"] -# def sen_plusplus_test(): -# model_d = "ies_10par_xsec" +def sen_plusplus_test(): + model_d = "ies_10par_xsec" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_sen_plusplus") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) -# pst.pestpp_options = {} -# pst.pestpp_options["gsa_method"] = "morris" -# pst.pestpp_options["gsa_sobol_samples"] = 50 -# pst.pestpp_options["gsa_sobol_par_dist"] = "unif" -# pst.pestpp_options["gsa_morris_r"] = 4 -# pst.pestpp_options["gsa_morris_p"] = 5 -# pst.pestpp_options["gsa_morris_delta"] = 2 -# pst.write(os.path.join(t_d,"pest_sen.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "pest_sen.pst", 5, master_dir=m_d, -# worker_root=model_d,port=port) - -# def secondary_marker_test(): -# t_d = os.path.join("secondary_marker_test","template") -# tpl_file = os.path.join(t_d,"par.dat.tpl") - -# with open(tpl_file,'w') as f: -# f.write("ptf ~\n") -# f.write("~ p1 ~\n") - -# tpl_file = "par.dat.tpl" -# b_d = os.getcwd() -# os.chdir(t_d) -# try: - -# ins_files = [f for f in os.listdir(".") if f.endswith(".ins")] -# with open("forward_run.py",'w') as f: -# f.write("import shutil\n") -# for ins_file in ins_files: -# out_file = ins_file.replace(".ins","") -# f.write("shutil.copy2('{0}','{1}')\n".format(out_file+"_bak",out_file)) - -# for ins_file in ins_files: - -# shutil.copy2(out_file+"_bak",out_file) -# pst = pyemu.Pst.from_io_files(tpl_file,tpl_file.replace(".tpl",""), -# ins_file,ins_file.replace(".ins","")) -# pst.control_data.noptmax = 0 -# pst.pestpp_options["additional_ins_delimiters"] = "|" -# pst.model_command = "python forward_run.py" -# pst.write(os.path.join("test.pst")) - -# pyemu.os_utils.run("{0} test.pst".format(exe_path)) -# pst = pyemu.Pst("test.pst") -# assert pst.res is not None -# d = pst.res.loc[pst.obs_names,"modelled"] - pst.observation_data.loc[pst.obs_names,"obsval"] -# l2_d = (d.apply(np.abs)**2).sum() + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_sen_plusplus") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) + pst.pestpp_options = {} + pst.pestpp_options["gsa_method"] = "morris" + pst.pestpp_options["gsa_sobol_samples"] = 50 + pst.pestpp_options["gsa_sobol_par_dist"] = "unif" + pst.pestpp_options["gsa_morris_r"] = 4 + pst.pestpp_options["gsa_morris_p"] = 5 + pst.pestpp_options["gsa_morris_delta"] = 2 + pst.write(os.path.join(t_d,"pest_sen.pst")) + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "pest_sen.pst", 5, master_dir=m_d, + worker_root=model_d,port=port) + +def secondary_marker_test(): + t_d = os.path.join("secondary_marker_test","template") + tpl_file = os.path.join(t_d,"par.dat.tpl") + + with open(tpl_file,'w') as f: + f.write("ptf ~\n") + f.write("~ p1 ~\n") + + tpl_file = "par.dat.tpl" + b_d = os.getcwd() + os.chdir(t_d) + try: + + ins_files = [f for f in os.listdir(".") if f.endswith(".ins")] + with open("forward_run.py",'w') as f: + f.write("import shutil\n") + for ins_file in ins_files: + out_file = ins_file.replace(".ins","") + f.write("shutil.copy2('{0}','{1}')\n".format(out_file+"_bak",out_file)) + + for ins_file in ins_files: + + shutil.copy2(out_file+"_bak",out_file) + pst = pyemu.Pst.from_io_files(tpl_file,tpl_file.replace(".tpl",""), + ins_file,ins_file.replace(".ins","")) + pst.control_data.noptmax = 0 + pst.pestpp_options["additional_ins_delimiters"] = "|" + pst.model_command = "python forward_run.py" + pst.write(os.path.join("test.pst")) + + pyemu.os_utils.run("{0} test.pst".format(exe_path)) + pst = pyemu.Pst("test.pst") + assert pst.res is not None + d = pst.res.loc[pst.obs_names,"modelled"] - pst.observation_data.loc[pst.obs_names,"obsval"] + l2_d = (d.apply(np.abs)**2).sum() -# except Exception as e: -# os.chdir(b_d) -# raise Exception(e) -# os.chdir(b_d) - -# def sen_basic_test(): -# model_d = "sen_invest" -# t_d = os.path.join(model_d, "template") -# if os.path.exists(t_d): -# shutil.rmtree(t_d) -# os.makedirs(t_d) -# par_names = ["p1","p2"] -# obs_names = ["p1","p2","p1+p2","p1*p2","p1^p2","const"] - -# tpl_file = os.path.join(t_d,"in.dat.tpl") -# with open(tpl_file,'w') as f: -# f.write("ptf ~\n") -# for par_name in par_names: -# f.write("{0} ~ {0} ~\n".format(par_name)) -# ins_file = os.path.join(t_d,"out.dat.ins") -# with open(ins_file,'w') as f: -# f.write("pif ~\n") -# for obs_name in obs_names: -# f.write("l1 w !{0}!\n".format(obs_name)) - -# with open(os.path.join(t_d,"forward_run.py"),'w') as f: -# f.write("import pandas as pd\n") -# f.write("df = pd.read_csv('in.dat',index_col=0,delim_whitespace=True,names=['name','value'])\n") -# f.write("df.loc['p1+p2','value'] = df.loc['p1','value'] + df.loc['p2','value']\n") -# f.write("df.loc['p1*p2','value'] = df.loc['p1','value'] * df.loc['p2','value']\n") -# f.write("df.loc['p1^p2','value'] = df.loc['p1','value'] * df.loc['p2','value']\n") -# f.write("df.loc['const','value'] = 1.0\n") -# f.write("df.to_csv('out.dat',sep=' ',header=False)\n") - -# with open(os.path.join(t_d,"in.dat"),'w') as f: -# f.write("p1 1.0\n") -# f.write("p2 1.0\n") -# f.write("p3 1.0\n") -# pyemu.os_utils.run("python forward_run.py",cwd=t_d) - -# pst = pyemu.Pst.from_io_files(tpl_files=tpl_file,in_files=tpl_file.replace(".tpl",""), -# ins_files=ins_file,out_files=ins_file.replace(".ins",""),pst_path=".") -# pst.model_command = "python forward_run.py" -# pst.control_data.noptmax = 0 -# pst.parameter_data.loc[:,"partrans"] = "log" -# pst.parameter_data.loc[:,"parchglim"] = "relative" -# pst.parameter_data.loc[:,"parubnd"] = 10.0 -# pst.parameter_data.loc[:,"parlbnd"] = .1 -# pst.parameter_data.loc[:,"parval1"] = 1.0 - -# msn_file = os.path.join(t_d,"pest.msn") -# mio_file = os.path.join(t_d, "pest.mio") - -# obs = pst.observation_data -# obs.loc[:, "weight"] = 0.0 -# obs.loc["const", "weight"] = 1.0 -# pst.write(os.path.join(t_d, "pest.pst")) -# pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies", "-sen")), cwd=t_d) -# df = pd.read_csv(msn_file, index_col=0) -# df.columns = df.columns.map(str.lower) -# df.columns = df.columns.map(str.strip) -# df.index = df.index.map(str.lower) -# print(df) -# assert df.sen_mean_abs.sum() == 0.0 -# assert df.sen_std_dev.sum() == 0.0 -# df = pd.read_csv(mio_file, index_col=0) -# df.columns = df.columns.map(str.lower) - -# df.loc[:, "parameter_name"] = df.parameter_name.apply(str.lower) -# df.index = df.index.map(str.lower) -# print(df) -# assert df.loc[df.parameter_name == "p2", :].loc["p1", "sen_mean_abs"] == 0 -# assert df.loc[df.parameter_name == "p1", :].loc["p2", "sen_mean_abs"] == 0 - -# pst.pestpp_options["gsa_method"] = "sobol" -# pst.pestpp_options["gsa_sobol_samples"] = 5 -# pst.write(os.path.join(t_d, "pest.pst")) -# #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies", "-sen")), cwd=t_d) -# m_d = os.path.join(model_d,"master_sobol") -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-sen"), "pest.pst", 5, master_dir=m_d, -# worker_root=model_d, port=port) -# si_vals = pd.read_csv(os.path.join(m_d,"pest.sobol.si.csv"),index_col=0) -# sti_vals = pd.read_csv(os.path.join(m_d,"pest.sobol.sti.csv"),index_col=0) -# v_d = os.path.join("sen_invest","verf") -# si_verf_vals = pd.read_csv(os.path.join(v_d, "si.csv"), index_col=0) -# sti_verf_vals = pd.read_csv(os.path.join(v_d, "sti.csv"), index_col=0) -# d_si = (si_vals.loc[pst.obs_names,:] - si_verf_vals.loc[pst.obs_names,:]).apply(np.abs) -# print(d_si.max()) -# assert d_si.max().max() < .001 -# d_sti = (sti_vals.loc[pst.obs_names, :] - sti_verf_vals.loc[pst.obs_names, :]).apply(np.abs) -# print(d_sti.max()) -# assert d_sti.max().max() < .001 - - -# def salib_verf(): -# import pyemu -# from SALib.sample import saltelli -# from SALib.analyze import sobol -# m_d = os.path.join("sen_invest","master_sobol") -# v_d = os.path.join("sen_invest","verf") -# if os.path.exists(v_d): -# shutil.rmtree(v_d) -# os.makedirs(v_d) -# pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) -# pst.add_transform_columns() -# bounds = [[l,u] for l,u in zip(pst.parameter_data.parlbnd_trans,pst.parameter_data.parubnd_trans)] -# problem = {"num_vars":pst.npar_adj,"names":pst.par_names,"bounds":bounds} -# test = saltelli.sample(problem,100,calc_second_order=False) -# out_df = pd.read_csv(os.path.join(m_d,"pest.sobol.obs.csv"),index_col=0) -# reorder_df = out_df.copy() -# idx = [0,3,1,2] -# for i in range(4): -# s = i*5 -# e = s + 5 -# chunk = out_df.iloc[s:e,:].copy() -# reorder_df.iloc[idx[i]::4,:] = chunk.values -# print(chunk.p1,reorder_df.iloc[idx[i]::4,:].p1) -# pass -# si_vals = pd.DataFrame(columns=pst.par_names,index=pst.obs_names) -# sti_vals = pd.DataFrame(columns=pst.par_names, index=pst.obs_names) - -# for obs_name in pst.obs_names: -# #if obs_name != "p1": -# # continue -# si = sobol.analyze(problem,reorder_df.loc[:,obs_name].values,calc_second_order=False,num_resamples=5) -# print(obs_name,si) -# si_vals.loc[obs_name,:] = si["S1"] -# sti_vals.loc[obs_name, :] = si["ST"] -# si_vals.to_csv(os.path.join(v_d,"si.csv")) -# sti_vals.to_csv(os.path.join(v_d, "sti.csv")) - -# # in_df = pd.read_csv(os.path.join(m_d,"pest.sobol.par.csv"),index_col=0) -# # import matplotlib.pyplot as plt -# # fig, ax = plt.subplots(1,1) -# # print(test) -# # test = test ** 10 -# # print(test) -# # ax.scatter(test[:,0],test[:,1],marker='.',color='g') -# # ax.scatter(in_df.iloc[:,0],in_df.iloc[:,1],marker='.',color='r') -# # -# # -# # plt.show() - - -# def tplins1_test(): -# model_d = "tplins_test_1" -# t_d = os.path.join(model_d, "test") -# if os.path.exists(t_d): -# shutil.rmtree(t_d) -# shutil.copytree(os.path.join(model_d,"template"),t_d) -# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) -# ins_file = os.path.join(t_d,"AOC_obs.txt.ins") -# pst.add_observations(ins_file,ins_file.replace(".ins",""),pst_path=".") -# pst.parameter_data.loc["k_10","parval1"] = 12345 -# pst.parameter_data.loc["k_10","parubnd"] = 200000 -# pst.pestpp_options["tpl_force_decimal"] = True -# pst.control_data.noptmax = 0 -# pst.write(os.path.join(t_d,"pest.pst")) -# pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=t_d) -# with open(os.path.join(t_d,"hk_Layer_1.ref"),'r') as f: -# for line in f: -# if "e" in line.lower(): -# raise Exception(line) -# pst.pestpp_options.pop("tpl_force_decimal") -# pst.control_data.noptmax = -1 -# pst.parameter_data.loc["k_10","parval1"] = 120 -# pst.parameter_data.loc["k_10","parubnd"] = 200 -# pst.write(os.path.join(t_d,"pest.pst")) + except Exception as e: + os.chdir(b_d) + raise Exception(e) + os.chdir(b_d) + +def sen_basic_test(): + model_d = "sen_invest" + t_d = os.path.join(model_d, "template") + if os.path.exists(t_d): + shutil.rmtree(t_d) + os.makedirs(t_d) + par_names = ["p1","p2"] + obs_names = ["p1","p2","p1+p2","p1*p2","p1^p2","const"] + + tpl_file = os.path.join(t_d,"in.dat.tpl") + with open(tpl_file,'w') as f: + f.write("ptf ~\n") + for par_name in par_names: + f.write("{0} ~ {0} ~\n".format(par_name)) + ins_file = os.path.join(t_d,"out.dat.ins") + with open(ins_file,'w') as f: + f.write("pif ~\n") + for obs_name in obs_names: + f.write("l1 w !{0}!\n".format(obs_name)) + + with open(os.path.join(t_d,"forward_run.py"),'w') as f: + f.write("import pandas as pd\n") + f.write("df = pd.read_csv('in.dat',index_col=0,delim_whitespace=True,names=['name','value'])\n") + f.write("df.loc['p1+p2','value'] = df.loc['p1','value'] + df.loc['p2','value']\n") + f.write("df.loc['p1*p2','value'] = df.loc['p1','value'] * df.loc['p2','value']\n") + f.write("df.loc['p1^p2','value'] = df.loc['p1','value'] * df.loc['p2','value']\n") + f.write("df.loc['const','value'] = 1.0\n") + f.write("df.to_csv('out.dat',sep=' ',header=False)\n") + + with open(os.path.join(t_d,"in.dat"),'w') as f: + f.write("p1 1.0\n") + f.write("p2 1.0\n") + f.write("p3 1.0\n") + pyemu.os_utils.run("python forward_run.py",cwd=t_d) + + pst = pyemu.Pst.from_io_files(tpl_files=tpl_file,in_files=tpl_file.replace(".tpl",""), + ins_files=ins_file,out_files=ins_file.replace(".ins",""),pst_path=".") + pst.model_command = "python forward_run.py" + pst.control_data.noptmax = 0 + pst.parameter_data.loc[:,"partrans"] = "log" + pst.parameter_data.loc[:,"parchglim"] = "relative" + pst.parameter_data.loc[:,"parubnd"] = 10.0 + pst.parameter_data.loc[:,"parlbnd"] = .1 + pst.parameter_data.loc[:,"parval1"] = 1.0 + + msn_file = os.path.join(t_d,"pest.msn") + mio_file = os.path.join(t_d, "pest.mio") + + obs = pst.observation_data + obs.loc[:, "weight"] = 0.0 + obs.loc["const", "weight"] = 1.0 + pst.write(os.path.join(t_d, "pest.pst")) + pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies", "-sen")), cwd=t_d) + df = pd.read_csv(msn_file, index_col=0) + df.columns = df.columns.map(str.lower) + df.columns = df.columns.map(str.strip) + df.index = df.index.map(str.lower) + print(df) + assert df.sen_mean_abs.sum() == 0.0 + assert df.sen_std_dev.sum() == 0.0 + df = pd.read_csv(mio_file, index_col=0) + df.columns = df.columns.map(str.lower) + + df.loc[:, "parameter_name"] = df.parameter_name.apply(str.lower) + df.index = df.index.map(str.lower) + print(df) + assert df.loc[df.parameter_name == "p2", :].loc["p1", "sen_mean_abs"] == 0 + assert df.loc[df.parameter_name == "p1", :].loc["p2", "sen_mean_abs"] == 0 + + pst.pestpp_options["gsa_method"] = "sobol" + pst.pestpp_options["gsa_sobol_samples"] = 5 + pst.write(os.path.join(t_d, "pest.pst")) + #pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies", "-sen")), cwd=t_d) + m_d = os.path.join(model_d,"master_sobol") + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-sen"), "pest.pst", 5, master_dir=m_d, + worker_root=model_d, port=port) + si_vals = pd.read_csv(os.path.join(m_d,"pest.sobol.si.csv"),index_col=0) + sti_vals = pd.read_csv(os.path.join(m_d,"pest.sobol.sti.csv"),index_col=0) + v_d = os.path.join("sen_invest","verf") + si_verf_vals = pd.read_csv(os.path.join(v_d, "si.csv"), index_col=0) + sti_verf_vals = pd.read_csv(os.path.join(v_d, "sti.csv"), index_col=0) + d_si = (si_vals.loc[pst.obs_names,:] - si_verf_vals.loc[pst.obs_names,:]).apply(np.abs) + print(d_si.max()) + assert d_si.max().max() < .001 + d_sti = (sti_vals.loc[pst.obs_names, :] - sti_verf_vals.loc[pst.obs_names, :]).apply(np.abs) + print(d_sti.max()) + assert d_sti.max().max() < .001 + + +def salib_verf(): + import pyemu + from SALib.sample import saltelli + from SALib.analyze import sobol + m_d = os.path.join("sen_invest","master_sobol") + v_d = os.path.join("sen_invest","verf") + if os.path.exists(v_d): + shutil.rmtree(v_d) + os.makedirs(v_d) + pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) + pst.add_transform_columns() + bounds = [[l,u] for l,u in zip(pst.parameter_data.parlbnd_trans,pst.parameter_data.parubnd_trans)] + problem = {"num_vars":pst.npar_adj,"names":pst.par_names,"bounds":bounds} + test = saltelli.sample(problem,100,calc_second_order=False) + out_df = pd.read_csv(os.path.join(m_d,"pest.sobol.obs.csv"),index_col=0) + reorder_df = out_df.copy() + idx = [0,3,1,2] + for i in range(4): + s = i*5 + e = s + 5 + chunk = out_df.iloc[s:e,:].copy() + reorder_df.iloc[idx[i]::4,:] = chunk.values + print(chunk.p1,reorder_df.iloc[idx[i]::4,:].p1) + pass + si_vals = pd.DataFrame(columns=pst.par_names,index=pst.obs_names) + sti_vals = pd.DataFrame(columns=pst.par_names, index=pst.obs_names) + + for obs_name in pst.obs_names: + #if obs_name != "p1": + # continue + si = sobol.analyze(problem,reorder_df.loc[:,obs_name].values,calc_second_order=False,num_resamples=5) + print(obs_name,si) + si_vals.loc[obs_name,:] = si["S1"] + sti_vals.loc[obs_name, :] = si["ST"] + si_vals.to_csv(os.path.join(v_d,"si.csv")) + sti_vals.to_csv(os.path.join(v_d, "sti.csv")) + + # in_df = pd.read_csv(os.path.join(m_d,"pest.sobol.par.csv"),index_col=0) + # import matplotlib.pyplot as plt + # fig, ax = plt.subplots(1,1) + # print(test) + # test = test ** 10 + # print(test) + # ax.scatter(test[:,0],test[:,1],marker='.',color='g') + # ax.scatter(in_df.iloc[:,0],in_df.iloc[:,1],marker='.',color='r') + # + # + # plt.show() + + +def tplins1_test(): + model_d = "tplins_test_1" + t_d = os.path.join(model_d, "test") + if os.path.exists(t_d): + shutil.rmtree(t_d) + shutil.copytree(os.path.join(model_d,"template"),t_d) + pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) + ins_file = os.path.join(t_d,"AOC_obs.txt.ins") + pst.add_observations(ins_file,ins_file.replace(".ins",""),pst_path=".") + pst.parameter_data.loc["k_10","parval1"] = 12345 + pst.parameter_data.loc["k_10","parubnd"] = 200000 + pst.pestpp_options["tpl_force_decimal"] = True + pst.control_data.noptmax = 0 + pst.write(os.path.join(t_d,"pest.pst")) + pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=t_d) + with open(os.path.join(t_d,"hk_Layer_1.ref"),'r') as f: + for line in f: + if "e" in line.lower(): + raise Exception(line) + pst.pestpp_options.pop("tpl_force_decimal") + pst.control_data.noptmax = -1 + pst.parameter_data.loc["k_10","parval1"] = 120 + pst.parameter_data.loc["k_10","parubnd"] = 200 + pst.write(os.path.join(t_d,"pest.pst")) -# pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) -# obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"]) -# obf_df.index = obf_df.obsnme -# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) -# res_df = pst.res + pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) + obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"]) + obf_df.index = obf_df.obsnme + pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) + res_df = pst.res -# d = (obf_df.obsval - res_df.modelled).apply(np.abs) -# #print(d) -# print(d.max()) -# assert d.max() < 1.0e-5, d - -# jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest.jcb")).to_dataframe().apply(np.abs) -# assert jco.sum().sum() == 0, jco.sum() - -# pst.control_data.noptmax = 0 -# pst.write(os.path.join(t_d,"pest.pst")) -# pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) - -# # check the input file - the last two number should be the same -# arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref")) -# assert arr[-2] == arr[-1],arr[-2] - arr[-1] - -# lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines() -# lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines() -# assert len(lines_tpl) - 1 == len(lines_in) - -# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) -# pst.control_data.noptmax = -1 -# pst.pestpp_options["fill_tpl_zeros"] = True -# pst.write(os.path.join(t_d,"pest_fill.pst")) -# pyemu.os_utils.run("{0} pest_fill.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) -# obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"]) -# obf_df.index = obf_df.obsnme -# pst = pyemu.Pst(os.path.join(t_d,"pest_fill.pst")) -# res_df = pst.res + d = (obf_df.obsval - res_df.modelled).apply(np.abs) + #print(d) + print(d.max()) + assert d.max() < 1.0e-5, d + + jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest.jcb")).to_dataframe().apply(np.abs) + assert jco.sum().sum() == 0, jco.sum() + + pst.control_data.noptmax = 0 + pst.write(os.path.join(t_d,"pest.pst")) + pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) + + # check the input file - the last two number should be the same + arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref")) + assert arr[-2] == arr[-1],arr[-2] - arr[-1] + + lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines() + lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines() + assert len(lines_tpl) - 1 == len(lines_in) + + pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) + pst.control_data.noptmax = -1 + pst.pestpp_options["fill_tpl_zeros"] = True + pst.write(os.path.join(t_d,"pest_fill.pst")) + pyemu.os_utils.run("{0} pest_fill.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) + obf_df = pd.read_csv(os.path.join(t_d,"out1.dat.obf"),delim_whitespace=True,header=None,names=["obsnme","obsval"]) + obf_df.index = obf_df.obsnme + pst = pyemu.Pst(os.path.join(t_d,"pest_fill.pst")) + res_df = pst.res -# d = (obf_df.obsval - res_df.modelled).apply(np.abs) -# #print(d) -# print(d.max()) -# assert d.max() < 1.0e-5, d - -# jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest_fill.jcb")).to_dataframe().apply(np.abs) -# assert jco.sum().sum() == 0, jco.sum() - -# pst.control_data.noptmax = 0 -# pst.write(os.path.join(t_d,"pest.pst")) -# pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) - -# # check the input file - the last two number should be the same -# arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref")) -# assert arr[-2] == arr[-1] - -# lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines() -# lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines() -# assert len(lines_tpl) - 1 == len(lines_in) - -# pst = pyemu.Pst(os.path.join(os.path.join(model_d,"template"), "pest.pst")) -# #pst.drop_observations(os.path.join(t_d,"AOC_obs.txt.ins"),pst_path=".") -# dum_obs = ['h01_03', 'h01_07', 'dummy_obs'] -# pst.observation_data.drop(index=dum_obs, inplace=True) -# pst.model_output_data = pd.DataFrame({"pest_file":"out1dum.dat.ins", -# "model_file":'out1.dat'},index=["out1dum.dat.ins"]) -# #pst.instruction_files = ['out1dum.dat.ins'] -# pst.control_data.noptmax = -1 -# pst.write(os.path.join(t_d, "pest_dum.pst")) -# pyemu.os_utils.run("{0} pest_dum.pst".format(exe_path.replace("-ies", "-glm")), cwd=t_d) -# obf_df = pd.read_csv(os.path.join(t_d, "out1.dat.obf"), delim_whitespace=True, header=None, -# names=["obsnme", "obsval"]) -# obf_df.index = obf_df.obsnme -# pst = pyemu.Pst(os.path.join(t_d, "pest_dum.pst")) -# res_df = pst.res - -# d = (obf_df.obsval - res_df.modelled).apply(np.abs) -# # print(d) -# print(d.max()) -# assert d.max() < 1.0e-5, d - -# jco = pyemu.Jco.from_binary(os.path.join(t_d, "pest_dum.jcb")).to_dataframe().apply(np.abs) -# assert jco.sum().sum() == 0, jco.sum() - - - -# def ext_stdcol_test(): -# model_d = "ies_10par_xsec" + d = (obf_df.obsval - res_df.modelled).apply(np.abs) + #print(d) + print(d.max()) + assert d.max() < 1.0e-5, d + + jco = pyemu.Jco.from_binary(os.path.join(t_d,"pest_fill.jcb")).to_dataframe().apply(np.abs) + assert jco.sum().sum() == 0, jco.sum() + + pst.control_data.noptmax = 0 + pst.write(os.path.join(t_d,"pest.pst")) + pyemu.os_utils.run("{0} pest.pst".format(exe_path.replace("-ies","-glm")),cwd=t_d) + + # check the input file - the last two number should be the same + arr = np.loadtxt(os.path.join(t_d,"hk_Layer_1.ref")) + assert arr[-2] == arr[-1] + + lines_tpl = open(os.path.join(t_d,"hk_Layer_1.ref.tpl"),'r').readlines() + lines_in = open(os.path.join(t_d,"hk_Layer_1.ref"),'r').readlines() + assert len(lines_tpl) - 1 == len(lines_in) + + pst = pyemu.Pst(os.path.join(os.path.join(model_d,"template"), "pest.pst")) + #pst.drop_observations(os.path.join(t_d,"AOC_obs.txt.ins"),pst_path=".") + dum_obs = ['h01_03', 'h01_07', 'dummy_obs'] + pst.observation_data.drop(index=dum_obs, inplace=True) + pst.model_output_data = pd.DataFrame({"pest_file":"out1dum.dat.ins", + "model_file":'out1.dat'},index=["out1dum.dat.ins"]) + #pst.instruction_files = ['out1dum.dat.ins'] + pst.control_data.noptmax = -1 + pst.write(os.path.join(t_d, "pest_dum.pst")) + pyemu.os_utils.run("{0} pest_dum.pst".format(exe_path.replace("-ies", "-glm")), cwd=t_d) + obf_df = pd.read_csv(os.path.join(t_d, "out1.dat.obf"), delim_whitespace=True, header=None, + names=["obsnme", "obsval"]) + obf_df.index = obf_df.obsnme + pst = pyemu.Pst(os.path.join(t_d, "pest_dum.pst")) + res_df = pst.res + + d = (obf_df.obsval - res_df.modelled).apply(np.abs) + # print(d) + print(d.max()) + assert d.max() < 1.0e-5, d + + jco = pyemu.Jco.from_binary(os.path.join(t_d, "pest_dum.jcb")).to_dataframe().apply(np.abs) + assert jco.sum().sum() == 0, jco.sum() + + + +def ext_stdcol_test(): + model_d = "ies_10par_xsec" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_ext_stdcol") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# shutil.copytree(t_d,m_d) + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_ext_stdcol") + if os.path.exists(m_d): + shutil.rmtree(m_d) + shutil.copytree(t_d,m_d) -# pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) -# obs = pst.observation_data -# obs.loc[pst.nnz_obs_names,"standard_deviation"] = 1/obs.loc[pst.nnz_obs_names,"weight"] -# pst.add_transform_columns() -# par = pst.parameter_data -# par.loc[pst.adj_par_names,"standard_deviation"] = (par.loc[pst.adj_par_names,"parubnd_trans"] - par.loc[pst.adj_par_names,"parlbnd_trans"]) / 4.0 -# #par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"] -# pst.pestpp_options["ies_num_reals"] = 10 -# pst.control_data.noptmax = -1 -# pst.write(os.path.join(m_d,"pest_base.pst")) -# pyemu.os_utils.run("{0} pest_base.pst".format(exe_path),cwd=m_d) - -# pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) -# pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) -# df1 = pd.read_csv(os.path.join(m_d,"pest_base.phi.meas.csv"),index_col=0) -# df2 = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.phi.meas.csv"),index_col=0) - -# d = (df1 - df2).apply(np.abs) -# print(d.max()) -# assert d.max().max() < 1.0e-6,d.max().max() - -# pst.pestpp_options["ies_num_reals"] = 100000 -# pst.control_data.noptmax = -2 -# obs = pst.observation_data -# obs.loc[pst.nnz_obs_names,"standard_deviation"] = 7.5 -# pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) -# pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) -# df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names] -# d = (df.std() - obs.loc[pst.nnz_obs_names,"standard_deviation"]).apply(np.abs) -# print(d) -# assert d.max() < 0.1,d.max() + pst = pyemu.Pst(os.path.join(m_d,"pest.pst")) + obs = pst.observation_data + obs.loc[pst.nnz_obs_names,"standard_deviation"] = 1/obs.loc[pst.nnz_obs_names,"weight"] + pst.add_transform_columns() + par = pst.parameter_data + par.loc[pst.adj_par_names,"standard_deviation"] = (par.loc[pst.adj_par_names,"parubnd_trans"] - par.loc[pst.adj_par_names,"parlbnd_trans"]) / 4.0 + #par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"] + pst.pestpp_options["ies_num_reals"] = 10 + pst.control_data.noptmax = -1 + pst.write(os.path.join(m_d,"pest_base.pst")) + pyemu.os_utils.run("{0} pest_base.pst".format(exe_path),cwd=m_d) + + pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) + pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) + df1 = pd.read_csv(os.path.join(m_d,"pest_base.phi.meas.csv"),index_col=0) + df2 = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.phi.meas.csv"),index_col=0) + + d = (df1 - df2).apply(np.abs) + print(d.max()) + assert d.max().max() < 1.0e-6,d.max().max() + + pst.pestpp_options["ies_num_reals"] = 100000 + pst.control_data.noptmax = -2 + obs = pst.observation_data + obs.loc[pst.nnz_obs_names,"standard_deviation"] = 7.5 + pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) + pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) + df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names] + d = (df.std() - obs.loc[pst.nnz_obs_names,"standard_deviation"]).apply(np.abs) + print(d) + assert d.max() < 0.1,d.max() -# obs = pst.observation_data -# obs.loc[pst.nnz_obs_names,"upper_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 1.1 -# obs.loc[pst.nnz_obs_names,"lower_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 0.9 -# par = pst.parameter_data -# par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"] -# pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) -# pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) -# df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names] -# mn = df.min() -# mx = df.max() -# dmn = mn - obs.loc[pst.nnz_obs_names,"obsval"] * 0.9 -# print(obs.loc[pst.nnz_obs_names,"obsval"] * 0.9) -# print(mn) -# print(dmn) -# dmx = mx - obs.loc[pst.nnz_obs_names,"obsval"] * 1.1 -# print(obs.loc[pst.nnz_obs_names,"obsval"] * 1.1) -# print(mx) -# print(dmx) - -# dmn = dmn.apply(np.abs) -# dmx = dmx.apply(np.abs) - -# assert dmn.max() < 1.0e-6,dmn -# assert dmx.max() < 1.0e-6,dmx - - -# def mf6_v5_ies_test(): -# model_d = "mf6_freyberg" - -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_ies_glm_loc") -# #if os.path.exists(m_d): -# # shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst")) -# pst.control_data.noptmax = 0 -# pst.write(os.path.join(t_d,"freyberg6_run_ies.pst")) -# pyemu.os_utils.run("{0} freyberg6_run_ies.pst".format(exe_path),cwd=t_d) - -# pst.control_data.noptmax = 3 -# par = pst.parameter_data - -# eff_lb = (par.parlbnd + (np.abs(par.parlbnd.values)*.01)).to_dict() -# eff_ub = (par.parubnd - (np.abs(par.parlbnd.values)*.01)).to_dict() -# log_idx = par.partrans.apply(lambda x: x=="log").to_dict() -# for p,log in log_idx.items(): -# if log: -# lb = np.log10(par.loc[p,"parlbnd"]) -# eff_lb[p] = (lb + (np.abs(lb)*.01)) -# ub = np.log10(par.loc[p,"parubnd"]) -# eff_ub[p] = (ub - (np.abs(ub)*.01)) - -# pargp_map = par.groupby(par.pargp).groups -# print(pargp_map) + obs = pst.observation_data + obs.loc[pst.nnz_obs_names,"upper_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 1.1 + obs.loc[pst.nnz_obs_names,"lower_bound"] = obs.loc[pst.nnz_obs_names,"obsval"] * 0.9 + par = pst.parameter_data + par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"] + pst.write(os.path.join(m_d,"pest_ext_stdcol.pst"),version=2) + pyemu.os_utils.run("{0} pest_ext_stdcol.pst".format(exe_path),cwd=m_d) + df = pd.read_csv(os.path.join(m_d,"pest_ext_stdcol.obs+noise.csv"),index_col=0).loc[:,pst.nnz_obs_names] + mn = df.min() + mx = df.max() + dmn = mn - obs.loc[pst.nnz_obs_names,"obsval"] * 0.9 + print(obs.loc[pst.nnz_obs_names,"obsval"] * 0.9) + print(mn) + print(dmn) + dmx = mx - obs.loc[pst.nnz_obs_names,"obsval"] * 1.1 + print(obs.loc[pst.nnz_obs_names,"obsval"] * 1.1) + print(mx) + print(dmx) + + dmn = dmn.apply(np.abs) + dmx = dmx.apply(np.abs) + + assert dmn.max() < 1.0e-6,dmn + assert dmx.max() < 1.0e-6,dmx + + +def mf6_v5_ies_test(): + model_d = "mf6_freyberg" + + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_ies_glm_loc") + #if os.path.exists(m_d): + # shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst")) + pst.control_data.noptmax = 0 + pst.write(os.path.join(t_d,"freyberg6_run_ies.pst")) + pyemu.os_utils.run("{0} freyberg6_run_ies.pst".format(exe_path),cwd=t_d) + + pst.control_data.noptmax = 3 + par = pst.parameter_data + + eff_lb = (par.parlbnd + (np.abs(par.parlbnd.values)*.01)).to_dict() + eff_ub = (par.parubnd - (np.abs(par.parlbnd.values)*.01)).to_dict() + log_idx = par.partrans.apply(lambda x: x=="log").to_dict() + for p,log in log_idx.items(): + if log: + lb = np.log10(par.loc[p,"parlbnd"]) + eff_lb[p] = (lb + (np.abs(lb)*.01)) + ub = np.log10(par.loc[p,"parubnd"]) + eff_ub[p] = (ub - (np.abs(ub)*.01)) + + pargp_map = par.groupby(par.pargp).groups + print(pargp_map) -# m_d = os.path.join(model_d, "master_ies_glm_noloc_standard") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) -# pst.pestpp_options.pop("ies_localizer",None) -# pst.pestpp_options.pop("ies_autoadaloc",None) -# pst.pestpp_options["ies_bad_phi_sigma"] = 2.5 -# pst.pestpp_options["ies_num_reals"] = 30 -# pst.pestpp_options["ensemble_output_precision"] = 40 -# pst.control_data.noptmax = 3 -# pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" -# pst.write(os.path.join(t_d, pst_name)) -# pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, -# master_dir=m_d, worker_root=model_d, port=port) -# phidf = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".phi.actual.csv"))) -# assert phidf.shape[0] == pst.control_data.noptmax + 1 -# for i in range(1,pst.control_data.noptmax+1): -# pcs = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.pcs.csv".format(i))),index_col=0) -# #print(pcs) -# pe = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.par.csv".format(i))),index_col=0) -# print(pe.shape) -# #print(pe) -# groups = pcs.index.values.copy() -# groups.sort() -# for group in groups: -# pnames = pargp_map[group].values -# lb_count,ub_count = 0,0 -# for pname in pnames: -# lb,ub = eff_lb[pname],eff_ub[pname] -# v = pe.loc[:,pname].values.copy() -# if log_idx[pname]: -# v = np.log10(v) -# low = np.zeros_like(v,dtype=int) -# low[v < lb] = 1 -# high = np.zeros_like(v,dtype=int) -# high[v > ub] = 1 -# lb_count += low.sum() -# ub_count += high.sum() -# print(i,group,len(pnames),lb_count,pcs.loc[group,"num_at_near_lbound"],ub_count,pcs.loc[group,"num_at_near_ubound"]) -# assert lb_count == pcs.loc[group,"num_at_near_lbound"] -# assert ub_count == pcs.loc[group,"num_at_near_ubound"] + m_d = os.path.join(model_d, "master_ies_glm_noloc_standard") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) + pst.pestpp_options.pop("ies_localizer",None) + pst.pestpp_options.pop("ies_autoadaloc",None) + pst.pestpp_options["ies_bad_phi_sigma"] = 2.5 + pst.pestpp_options["ies_num_reals"] = 30 + pst.pestpp_options["ensemble_output_precision"] = 40 + pst.control_data.noptmax = 3 + pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" + pst.write(os.path.join(t_d, pst_name)) + pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, + master_dir=m_d, worker_root=model_d, port=port) + phidf = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".phi.actual.csv"))) + assert phidf.shape[0] == pst.control_data.noptmax + 1 + for i in range(1,pst.control_data.noptmax+1): + pcs = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.pcs.csv".format(i))),index_col=0) + #print(pcs) + pe = pd.read_csv(os.path.join(m_d,pst_name.replace(".pst",".{0}.par.csv".format(i))),index_col=0) + print(pe.shape) + #print(pe) + groups = pcs.index.values.copy() + groups.sort() + for group in groups: + pnames = pargp_map[group].values + lb_count,ub_count = 0,0 + for pname in pnames: + lb,ub = eff_lb[pname],eff_ub[pname] + v = pe.loc[:,pname].values.copy() + if log_idx[pname]: + v = np.log10(v) + low = np.zeros_like(v,dtype=int) + low[v < lb] = 1 + high = np.zeros_like(v,dtype=int) + high[v > ub] = 1 + lb_count += low.sum() + ub_count += high.sum() + print(i,group,len(pnames),lb_count,pcs.loc[group,"num_at_near_lbound"],ub_count,pcs.loc[group,"num_at_near_ubound"]) + assert lb_count == pcs.loc[group,"num_at_near_lbound"] + assert ub_count == pcs.loc[group,"num_at_near_ubound"] -# pst.write(os.path.join(t_d,"freyberg6_run_ies_glm_loc.pst")) - -# m_d = os.path.join(model_d, "master_ies_glm_covloc") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst.pestpp_options["ies_loc_type"] = "cov" -# pst.pestpp_options["ies_lambda_mults"] = [1.0] -# pst.pestpp_options["lambda_scale_fac"] = [1.0] -# pst.control_data.noptmax = 2 -# #pst.pestpp_options.pop("ies_localizer",None) -# pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_covloc.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_covloc.pst", num_workers=15, -# master_dir=m_d, worker_root=model_d, port=port) - -# m_d = os.path.join(model_d, "master_ies_glm_noloc") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) -# pst.pestpp_options.pop("ies_localizer",None) -# pst.pestpp_options.pop("ies_autoadaloc",None) -# pst.pestpp_options["ies_lambda_mults"] = [1.0] -# pst.pestpp_options["lambda_scale_fac"] = [1.0] -# pst.control_data.noptmax = 2 -# pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc.pst", num_workers=15, -# master_dir=m_d, worker_root=model_d, port=port) - -# m_d = os.path.join(model_d, "master_ies_mda_loc") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) -# pst.pestpp_options["ies_lambda_mults"] = [1.0] -# pst.pestpp_options["lambda_scale_fac"] = [1.0] -# pst.control_data.noptmax = 2 -# pst.pestpp_options["ies_use_mda"] = True -# pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_loc.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_loc.pst", num_workers=15, -# master_dir=m_d, worker_root=model_d, port=port) + pst.write(os.path.join(t_d,"freyberg6_run_ies_glm_loc.pst")) + + m_d = os.path.join(model_d, "master_ies_glm_covloc") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst.pestpp_options["ies_loc_type"] = "cov" + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] + pst.control_data.noptmax = 2 + #pst.pestpp_options.pop("ies_localizer",None) + pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_covloc.pst")) + pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_covloc.pst", num_workers=15, + master_dir=m_d, worker_root=model_d, port=port) + + m_d = os.path.join(model_d, "master_ies_glm_noloc") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) + pst.pestpp_options.pop("ies_localizer",None) + pst.pestpp_options.pop("ies_autoadaloc",None) + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] + pst.control_data.noptmax = 2 + pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc.pst")) + pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc.pst", num_workers=15, + master_dir=m_d, worker_root=model_d, port=port) + + m_d = os.path.join(model_d, "master_ies_mda_loc") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] + pst.control_data.noptmax = 2 + pst.pestpp_options["ies_use_mda"] = True + pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_loc.pst")) + pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_loc.pst", num_workers=15, + master_dir=m_d, worker_root=model_d, port=port) -# # m_d = os.path.join(model_d, "master_ies_mda_covloc") -# # if os.path.exists(m_d): -# # shutil.rmtree(m_d) -# # pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) -# # pst.control_data.noptmax = 3 -# # pst.pestpp_options["ies_use_mda"] = True -# # pst.pestpp_options["ies_loc_type"] = "cov" -# # pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_covloc.pst")) -# # pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_covloc.pst", num_workers=15, -# # master_dir=m_d, worker_root=model_d, port=port) + # m_d = os.path.join(model_d, "master_ies_mda_covloc") + # if os.path.exists(m_d): + # shutil.rmtree(m_d) + # pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) + # pst.control_data.noptmax = 3 + # pst.pestpp_options["ies_use_mda"] = True + # pst.pestpp_options["ies_loc_type"] = "cov" + # pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_covloc.pst")) + # pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_covloc.pst", num_workers=15, + # master_dir=m_d, worker_root=model_d, port=port) -# m_d = os.path.join(model_d, "master_ies_mda_noloc") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) -# pst.control_data.noptmax = 2 -# pst.pestpp_options["ies_lambda_mults"] = [1.0] -# pst.pestpp_options["lambda_scale_fac"] = [1.0] -# pst.pestpp_options["ies_use_mda"] = True -# pst.pestpp_options.pop("ies_localizer", None) -# pst.pestpp_options.pop("ies_autoadaloc", None) -# pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_noloc.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_noloc.pst", num_workers=15, -# master_dir=m_d, worker_root=model_d, port=port) - -# m_d = os.path.join(model_d, "master_ies_glm_loc_mm") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) -# pst.control_data.noptmax = 2 -# pst.pestpp_options["ies_lambda_mults"] = [1.0] -# pst.pestpp_options["lambda_scale_fac"] = [1.0] -# pst.pestpp_options["ies_num_threads"] = 1 -# pst.pestpp_options["ies_use_mda"] = False -# pst.pestpp_options.pop("ies_localizer", None) -# pst.pestpp_options.pop("ies_autoadaloc", None) -# pst.pestpp_options["ies_multimodal_alpha"] = 0.1 -# pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_loc_mm.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_loc_mm.pst", num_workers=15, -# master_dir=m_d, worker_root=model_d, port=port) - -# m_d = os.path.join(model_d, "master_ies_glm_noloc_mm") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) -# pst.control_data.noptmax = 2 -# pst.pestpp_options["ies_use_mda"] = False -# pst.pestpp_options["ies_lambda_mults"] = [1.0] -# pst.pestpp_options["lambda_scale_fac"] = [1.0] -# pst.pestpp_options.pop("ies_localizer", None) -# pst.pestpp_options.pop("ies_autoadaloc", None) -# pst.pestpp_options["ies_multimodal_alpha"] = 0.25 -# pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc_mm.pst")) -# pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc_mm.pst", num_workers=15, -# master_dir=m_d, worker_root=model_d, port=port) - - - - -# def mf6_v5_sen_test(): - -# model_d = "mf6_freyberg" - -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_sen") -# #if os.path.exists(m_d): -# # shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_sen.pst")) -# pst.pestpp_options["panther_transfer_on_finish"] = ["freyberg6_freyberg.cbc","freyberg6.lst","ies_prior.jcb"] -# pst.write(os.path.join(t_d,"freyberg6_run_sen_trn.pst")) -# m_d = os.path.join(model_d,"master_sen") -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "freyberg6_run_sen_trn.pst", -# num_workers=15, worker_root=model_d, -# port=4004,verbose=True,master_dir=m_d) - -# pst = pyemu.Pst(os.path.join(m_d,"freyberg6_run_sen_trn.pst")) -# mio_file = os.path.join(m_d,"freyberg6_run_sen_trn.mio") -# assert os.path.exists(mio_file),mio_file -# df = pd.read_csv(mio_file) -# assert df.shape[0] > 1 -# msn_file = mio_file.replace(".mio",".msn") -# assert os.path.exists(msn_file),msn_file -# msngrp_file = msn_file.replace(".msn",".group.msn") -# assert os.path.exists(msngrp_file),msngrp_file - -# jcb_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".jcb")] -# print(len(jcb_files)) -# assert len(jcb_files) == 52 -# for jcb_file in jcb_files: -# j = pyemu.Jco.from_binary(os.path.join(m_d,jcb_file)) - -# lst_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".lst")] -# print(len(lst_files)) -# assert len(lst_files) == 52 + m_d = os.path.join(model_d, "master_ies_mda_noloc") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) + pst.control_data.noptmax = 2 + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] + pst.pestpp_options["ies_use_mda"] = True + pst.pestpp_options.pop("ies_localizer", None) + pst.pestpp_options.pop("ies_autoadaloc", None) + pst.write(os.path.join(t_d, "freyberg6_run_ies_mda_noloc.pst")) + pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_mda_noloc.pst", num_workers=15, + master_dir=m_d, worker_root=model_d, port=port) + + m_d = os.path.join(model_d, "master_ies_glm_loc_mm") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) + pst.control_data.noptmax = 2 + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] + pst.pestpp_options["ies_num_threads"] = 1 + pst.pestpp_options["ies_use_mda"] = False + pst.pestpp_options.pop("ies_localizer", None) + pst.pestpp_options.pop("ies_autoadaloc", None) + pst.pestpp_options["ies_multimodal_alpha"] = 0.1 + pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_loc_mm.pst")) + pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_loc_mm.pst", num_workers=15, + master_dir=m_d, worker_root=model_d, port=port) + + m_d = os.path.join(model_d, "master_ies_glm_noloc_mm") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d, "freyberg6_run_ies.pst")) + pst.control_data.noptmax = 2 + pst.pestpp_options["ies_use_mda"] = False + pst.pestpp_options["ies_lambda_mults"] = [1.0] + pst.pestpp_options["lambda_scale_fac"] = [1.0] + pst.pestpp_options.pop("ies_localizer", None) + pst.pestpp_options.pop("ies_autoadaloc", None) + pst.pestpp_options["ies_multimodal_alpha"] = 0.25 + pst.write(os.path.join(t_d, "freyberg6_run_ies_glm_noloc_mm.pst")) + pyemu.os_utils.start_workers(t_d, exe_path, "freyberg6_run_ies_glm_noloc_mm.pst", num_workers=15, + master_dir=m_d, worker_root=model_d, port=port) + + + + +def mf6_v5_sen_test(): + + model_d = "mf6_freyberg" + + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_sen") + #if os.path.exists(m_d): + # shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_sen.pst")) + pst.pestpp_options["panther_transfer_on_finish"] = ["freyberg6_freyberg.cbc","freyberg6.lst","ies_prior.jcb"] + pst.write(os.path.join(t_d,"freyberg6_run_sen_trn.pst")) + m_d = os.path.join(model_d,"master_sen") + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-sen"), "freyberg6_run_sen_trn.pst", + num_workers=15, worker_root=model_d, + port=4004,verbose=True,master_dir=m_d) + + pst = pyemu.Pst(os.path.join(m_d,"freyberg6_run_sen_trn.pst")) + mio_file = os.path.join(m_d,"freyberg6_run_sen_trn.mio") + assert os.path.exists(mio_file),mio_file + df = pd.read_csv(mio_file) + assert df.shape[0] > 1 + msn_file = mio_file.replace(".mio",".msn") + assert os.path.exists(msn_file),msn_file + msngrp_file = msn_file.replace(".msn",".group.msn") + assert os.path.exists(msngrp_file),msngrp_file + + jcb_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".jcb")] + print(len(jcb_files)) + assert len(jcb_files) == 52 + for jcb_file in jcb_files: + j = pyemu.Jco.from_binary(os.path.join(m_d,jcb_file)) + + lst_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx_") and f.lower().endswith(".lst")] + print(len(lst_files)) + assert len(lst_files) == 52 -# cbc_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx") and f.lower().endswith(".cbc")] -# print(len(cbc_files)) -# assert len(cbc_files) == 52 + cbc_files = [f for f in os.listdir(m_d) if f.lower().startswith("ftx") and f.lower().endswith(".cbc")] + print(len(cbc_files)) + assert len(cbc_files) == 52 -# def mf6_v5_opt_stack_test(): -# model_d = "mf6_freyberg" +def mf6_v5_opt_stack_test(): + model_d = "mf6_freyberg" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_opt_stack") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_opt.pst")) -# m_d = os.path.join(model_d,"master_opt_stack") -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-opt"), "freyberg6_run_opt.pst", -# num_workers=15, master_dir=m_d,worker_root=model_d, -# port=port) - -# assert os.path.exists(os.path.join(m_d,"freyberg6_run_opt.1.sim+chance.rei")) -# assert os.path.exists(os.path.join(m_d,"freyberg6_run_opt.1.obs_stack.csv")) - - -# def mf6_v5_glm_test(): -# model_d = "mf6_freyberg" + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_opt_stack") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_opt.pst")) + m_d = os.path.join(model_d,"master_opt_stack") + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies","-opt"), "freyberg6_run_opt.pst", + num_workers=15, master_dir=m_d,worker_root=model_d, + port=port) + + assert os.path.exists(os.path.join(m_d,"freyberg6_run_opt.1.sim+chance.rei")) + assert os.path.exists(os.path.join(m_d,"freyberg6_run_opt.1.obs_stack.csv")) + + +def mf6_v5_glm_test(): + model_d = "mf6_freyberg" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_glm") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_glm.pst")) -# m_d = os.path.join(model_d,"master_glm") -# pyemu.os_utils.start_workers(t_d, "pestpp-glm", "freyberg6_run_glm.pst", -# num_workers=15, master_dir=m_d,worker_root=model_d, -# port=port) - -# oe_file = os.path.join(m_d,"freyberg6_run_glm.post.obsen.csv") -# assert os.path.exists(oe_file) -# oe = pd.read_csv(oe_file) -# assert oe.shape[0] == pst.pestpp_options["glm_num_reals"],"{0},{1}".\ -# format(oe.shape[0],pst.pestpp_options["glm_num_reals"]) - - -# def cmdline_test(): -# model_d = "mf6_freyberg" + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_glm") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_glm.pst")) + m_d = os.path.join(model_d,"master_glm") + pyemu.os_utils.start_workers(t_d, "pestpp-glm", "freyberg6_run_glm.pst", + num_workers=15, master_dir=m_d,worker_root=model_d, + port=port) + + oe_file = os.path.join(m_d,"freyberg6_run_glm.post.obsen.csv") + assert os.path.exists(oe_file) + oe = pd.read_csv(oe_file) + assert oe.shape[0] == pst.pestpp_options["glm_num_reals"],"{0},{1}".\ + format(oe.shape[0],pst.pestpp_options["glm_num_reals"]) + + +def cmdline_test(): + model_d = "mf6_freyberg" -# t_d = os.path.join(model_d,"template") -# pst_name = "freyberg6_run_glm.pst" -# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_glm.pst")) -# pst.pestpp_options["debug_parse_only"] = True -# pst_name = "CmdLine_test.pst" #camel case on purpose for linux testing -# pst.write(os.path.join(t_d,pst_name)) -# pyemu.os_utils.run("{0} {1}".format(exe_path,pst_name),cwd=t_d) -# pyemu.os_utils.run("{0} {1} /h :4004".format(exe_path,pst_name),cwd=t_d) -# pyemu.os_utils.run("{0} {1} /r /h :4004".format(exe_path.replace("-ies","-glm"),pst_name),cwd=t_d) -# pyemu.os_utils.run("{0} {1} /r ".format(exe_path.replace("-ies","-glm"),pst_name),cwd=t_d) + t_d = os.path.join(model_d,"template") + pst_name = "freyberg6_run_glm.pst" + pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_glm.pst")) + pst.pestpp_options["debug_parse_only"] = True + pst_name = "CmdLine_test.pst" #camel case on purpose for linux testing + pst.write(os.path.join(t_d,pst_name)) + pyemu.os_utils.run("{0} {1}".format(exe_path,pst_name),cwd=t_d) + pyemu.os_utils.run("{0} {1} /h :4004".format(exe_path,pst_name),cwd=t_d) + pyemu.os_utils.run("{0} {1} /r /h :4004".format(exe_path.replace("-ies","-glm"),pst_name),cwd=t_d) + pyemu.os_utils.run("{0} {1} /r ".format(exe_path.replace("-ies","-glm"),pst_name),cwd=t_d) -# try: -# pyemu.os_utils.run("{0} {1} \\h :4004".format(exe_path,pst_name),cwd=t_d) + try: + pyemu.os_utils.run("{0} {1} \\h :4004".format(exe_path,pst_name),cwd=t_d) -# except: -# pass -# else: -# raise Exception("should have failed") + except: + pass + else: + raise Exception("should have failed") -# try: -# pyemu.os_utils.run("{0} {1} :4004".format(exe_path,pst_name),cwd=t_d) + try: + pyemu.os_utils.run("{0} {1} :4004".format(exe_path,pst_name),cwd=t_d) -# except: -# pass -# else: -# raise Exception("should have failed") + except: + pass + else: + raise Exception("should have failed") -# try: -# pyemu.os_utils.run("{0} {1} /h 4004".format(exe_path,pst_name),cwd=t_d) + try: + pyemu.os_utils.run("{0} {1} /h 4004".format(exe_path,pst_name),cwd=t_d) -# except: -# pass -# else: -# raise Exception("should have failed") + except: + pass + else: + raise Exception("should have failed") -# def fr_fail_test(): -# model_d = "ies_10par_xsec" -# base_d = os.path.join(model_d, "template") -# new_d = os.path.join(model_d, "test_template") -# if os.path.exists(new_d): -# shutil.rmtree(new_d) -# shutil.copytree(base_d, new_d) -# print(platform.platform().lower()) -# pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) -# with open(os.path.join(new_d,"run.py"),'w') as f: -# f.write("import pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\nprint(junk)\n") -# pst.model_command = "python run.py" -# oe_file = os.path.join(new_d, "pest.0.obs.csv") -# if os.path.exists(oe_file): -# os.remove(oe_file) -# pst.control_data.noptmax = 1 -# pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list" -# pst.pestpp_options["ies_num_reals"] = 10 -# #pst.pestpp_options["panther_agent_freeze_on_fail"] = True -# pst.write(os.path.join(new_d, "pest.pst")) -# try: -# pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d) -# except: -# pass -# else: -# raise Exception("should have failed") - -# assert not os.path.exists(oe_file) -# m_d = os.path.join(model_d,"fr_fail_master") -# try: -# pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d) -# except: -# pass -# else: -# raise Exception("should have failed") -# oe_file = os.path.join(m_d, "pest.0.obs.csv") -# assert not os.path.exists(oe_file) - -# trx_files = [f for f in os.listdir(m_d) if f.endswith(".list")] -# print(trx_files) -# assert len(trx_files) == 11,len(trx_files) - - - - - -# def sen_grp_test(): +def fr_fail_test(): + model_d = "ies_10par_xsec" + base_d = os.path.join(model_d, "template") + new_d = os.path.join(model_d, "test_template") + if os.path.exists(new_d): + shutil.rmtree(new_d) + shutil.copytree(base_d, new_d) + print(platform.platform().lower()) + pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) + with open(os.path.join(new_d,"run.py"),'w') as f: + f.write("import pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\nprint(junk)\n") + pst.model_command = "python run.py" + oe_file = os.path.join(new_d, "pest.0.obs.csv") + if os.path.exists(oe_file): + os.remove(oe_file) + pst.control_data.noptmax = 1 + pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list" + pst.pestpp_options["ies_num_reals"] = 10 + #pst.pestpp_options["panther_agent_freeze_on_fail"] = True + pst.write(os.path.join(new_d, "pest.pst")) + try: + pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=new_d) + except: + pass + else: + raise Exception("should have failed") + + assert not os.path.exists(oe_file) + m_d = os.path.join(model_d,"fr_fail_master") + try: + pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d) + except: + pass + else: + raise Exception("should have failed") + oe_file = os.path.join(m_d, "pest.0.obs.csv") + assert not os.path.exists(oe_file) + + trx_files = [f for f in os.listdir(m_d) if f.endswith(".list")] + print(trx_files) + assert len(trx_files) == 11,len(trx_files) + + + + + +def sen_grp_test(): -# model_d = "ies_10par_xsec" + model_d = "ies_10par_xsec" -# t_d = os.path.join(model_d, "template") -# m_d = os.path.join(model_d, "master_sen_group") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) + t_d = os.path.join(model_d, "template") + m_d = os.path.join(model_d, "master_sen_group") + if os.path.exists(m_d): + shutil.rmtree(m_d) -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-sen"), "pest.pst", 10, master_dir=m_d, -# worker_root=model_d, port=port) - -# msn_file = os.path.join(m_d,"pest.msn") -# msndf = pd.read_csv(msn_file) - -# grp_file = msn_file.replace(".msn",".group.msn") -# grpdf = pd.read_csv(grp_file) -# assert msndf.shape[0] == grpdf.shape[0] -# for col in ["sen_mean","sen_mean_abs","sen_std_dev"]: -# diff = np.abs(msndf.loc[:,col].sum() - grpdf.loc[:,col].sum()) -# print(col,diff) -# assert diff < 1.0e-6 - - -# def agnostic_path_test(): -# model_d = "ies_10par_xsec" - -# t_d = os.path.join(model_d, "template") -# m_d = os.path.join(model_d, "test_path") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# shutil.copytree(t_d,m_d) - -# pst = pyemu.Pst(os.path.join(m_d, "pest.pst")) -# pst.parameter_data.loc[pst.adj_par_names,"parval1"] = np.random.random(pst.npar_adj) -# pst.control_data.noptmax = 0 -# pst.write(os.path.join(m_d,"pest.pst")) -# pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=m_d) - -# arr1 = np.loadtxt(os.path.join(m_d,"hk_Layer_1.ref")) -# arr2 = np.loadtxt(os.path.join(m_d,"nested","really","deep","hk_Layer_1.ref")) -# d = np.abs(arr1-arr2).sum() -# print(d) -# assert d == 0,d - -# def fr_timeout_test(): -# model_d = "ies_10par_xsec" -# base_d = os.path.join(model_d, "template") -# new_d = os.path.join(model_d, "test_template") -# if os.path.exists(new_d): -# shutil.rmtree(new_d) -# shutil.copytree(base_d, new_d) -# print(platform.platform().lower()) -# pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) -# with open(os.path.join(new_d,"run.py"),'w') as f: -# f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n") -# f.write("if not os.path.exists('run.info'):\n exit()\n") -# f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n") -# f.write("if rnum % 2 == 0:\n time.sleep(10000000)\n") -# pst.model_command = "python run.py" -# oe_file = os.path.join(new_d, "pest.0.obs.csv") -# if os.path.exists(oe_file): -# os.remove(oe_file) -# pst.control_data.noptmax = -1 -# pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10 -# pst.pestpp_options["overdue_giveup_minutes"] = 0.25 -# pst.pestpp_options["ies_num_reals"] = 10 -# pst.pestpp_options["ies_include_base"] = False -# pst.pestpp_options["max_run_fail"] = 1 - -# #pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list" -# pst.pestpp_options["panther_agent_freeze_on_fail"] = False -# pst.write(os.path.join(new_d, "pest.pst")) - -# m_d = os.path.join(model_d,"fr_timeout_master") -# pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d) -# oe_file = os.path.join(m_d, "pest.0.obs.csv") -# assert os.path.exists(oe_file) -# oe = pd.read_csv(oe_file,index_col=0) -# print(oe.shape) -# assert oe.shape[0] == 5,oe.shape - -# with open(os.path.join(new_d,"run.py"),'w') as f: -# f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n") -# f.write("if not os.path.exists('run.info'):\n exit()\n") -# f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n") -# f.write("if rnum % 10 == 0:\n print(junk)\n") -# pst.pestpp_options = {} -# pst.pestpp_options["ies_num_reals"] = 20 # hard coded to conditional below -# pst.pestpp_options["panther_agent_freeze_on_fail"] = True -# #pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10 -# #pst.pestpp_options["overdue_giveup_minutes"] = 0.25 -# pst.write(os.path.join(new_d, "pest.pst")) -# pst.control_data.noptmax = 2 - -# pst.write(os.path.join(new_d, "pest.pst")) -# m_d = os.path.join(model_d,"fr_timeout_master_freeze") -# #num workers hard coded with conditional below -# pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=10,worker_root=model_d,master_dir=m_d) -# #df = pyemu.helpers.parse_rmr_file(os.path.join(m_d,"pest.rmr")) -# #print(df.action.to_list()) -# oe = pd.read_csv(os.path.join(m_d,"pest.{0}.obs.csv".format(pst.control_data.noptmax)),index_col=0) -# assert oe.shape[0] == 17 # hard coded to num reals -# with open(os.path.join(m_d,"pest.rmr"),'r') as f: -# for line in f: -# if "timeout" in line.lower(): -# raise Exception() -# if line.strip().lower().endswith("agents connected"): -# num = int(line.strip().split()[0]) -# print(line.strip()) -# assert num == 7 # hard coded above - - - -# def ins_missing_e_test(): -# import os -# import shutil -# import pyemu -# t_d = os.path.join("tplins_test_1","test_missing_e") -# if os.path.exists(t_d): -# shutil.rmtree(t_d) -# os.makedirs(t_d) -# bd = os.getcwd() -# os.chdir(t_d) -# with open("model.output.bak",'w') as f: -# f.write("12345-123\n") -# pst = pyemu.helpers.pst_from_parnames_obsnames(["p1"],["o1"]) -# pst.control_data.noptmax = 0 -# with open("forward_run.py",'w') as f: -# f.write("import shutil\n") -# f.write("shutil.copy2('model.output.bak','model.output')\n") -# pst.model_command = "python forward_run.py" -# pst.write("test.pst") -# os.chdir(bd) -# try: -# pyemu.os_utils.run("{0} test.pst".format(exe_path),cwd=t_d) -# except: -# pass -# else: -# raise Exception("should have failed") - - -# def prep_ends(): -# model_d = "mf6_freyberg" -# base_d = os.path.join(model_d, "template") -# new_d = os.path.join(model_d, "ends") -# if os.path.exists(new_d): -# shutil.rmtree(new_d) -# os.makedirs(new_d) -# skip = ["pst","csv","log","grb","hds","par","rei","lst","jcb","cov","rec","cbc"] -# files = [f for f in os.listdir(base_d) if f.lower().split('.')[-1] not in skip] -# print(files) -# [shutil.copy2(os.path.join(base_d,f),os.path.join(new_d,f)) for f in files] -# [shutil.copy2(os.path.join(base_d,f),os.path.join(new_d,f)) for f in ["ies_prior.jcb"]] + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-sen"), "pest.pst", 10, master_dir=m_d, + worker_root=model_d, port=port) + + msn_file = os.path.join(m_d,"pest.msn") + msndf = pd.read_csv(msn_file) + + grp_file = msn_file.replace(".msn",".group.msn") + grpdf = pd.read_csv(grp_file) + assert msndf.shape[0] == grpdf.shape[0] + for col in ["sen_mean","sen_mean_abs","sen_std_dev"]: + diff = np.abs(msndf.loc[:,col].sum() - grpdf.loc[:,col].sum()) + print(col,diff) + assert diff < 1.0e-6 + + +def agnostic_path_test(): + model_d = "ies_10par_xsec" + + t_d = os.path.join(model_d, "template") + m_d = os.path.join(model_d, "test_path") + if os.path.exists(m_d): + shutil.rmtree(m_d) + shutil.copytree(t_d,m_d) + + pst = pyemu.Pst(os.path.join(m_d, "pest.pst")) + pst.parameter_data.loc[pst.adj_par_names,"parval1"] = np.random.random(pst.npar_adj) + pst.control_data.noptmax = 0 + pst.write(os.path.join(m_d,"pest.pst")) + pyemu.os_utils.run("{0} pest.pst".format(exe_path),cwd=m_d) + + arr1 = np.loadtxt(os.path.join(m_d,"hk_Layer_1.ref")) + arr2 = np.loadtxt(os.path.join(m_d,"nested","really","deep","hk_Layer_1.ref")) + d = np.abs(arr1-arr2).sum() + print(d) + assert d == 0,d + +def fr_timeout_test(): + model_d = "ies_10par_xsec" + base_d = os.path.join(model_d, "template") + new_d = os.path.join(model_d, "test_template") + if os.path.exists(new_d): + shutil.rmtree(new_d) + shutil.copytree(base_d, new_d) + print(platform.platform().lower()) + pst = pyemu.Pst(os.path.join(new_d, "pest.pst")) + with open(os.path.join(new_d,"run.py"),'w') as f: + f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n") + f.write("if not os.path.exists('run.info'):\n exit()\n") + f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n") + f.write("if rnum % 2 == 0:\n time.sleep(10000000)\n") + pst.model_command = "python run.py" + oe_file = os.path.join(new_d, "pest.0.obs.csv") + if os.path.exists(oe_file): + os.remove(oe_file) + pst.control_data.noptmax = -1 + pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10 + pst.pestpp_options["overdue_giveup_minutes"] = 0.25 + pst.pestpp_options["ies_num_reals"] = 10 + pst.pestpp_options["ies_include_base"] = False + pst.pestpp_options["max_run_fail"] = 1 + + #pst.pestpp_options["panther_transfer_on_fail"] = "10par_xsec.list" + pst.pestpp_options["panther_agent_freeze_on_fail"] = False + pst.write(os.path.join(new_d, "pest.pst")) + + m_d = os.path.join(model_d,"fr_timeout_master") + pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=5,worker_root=model_d,master_dir=m_d) + oe_file = os.path.join(m_d, "pest.0.obs.csv") + assert os.path.exists(oe_file) + oe = pd.read_csv(oe_file,index_col=0) + print(oe.shape) + assert oe.shape[0] == 5,oe.shape + + with open(os.path.join(new_d,"run.py"),'w') as f: + f.write("import os\nimport time\nimport pyemu\npyemu.os_utils.run('mfnwt 10par_xsec.nam')\n") + f.write("if not os.path.exists('run.info'):\n exit()\n") + f.write("lines = open('run.info','r').readlines()\nrnum = int(lines[-1].split()[-1].split(':')[-1])\n") + f.write("if rnum % 10 == 0:\n print(junk)\n") + pst.pestpp_options = {} + pst.pestpp_options["ies_num_reals"] = 20 # hard coded to conditional below + pst.pestpp_options["panther_agent_freeze_on_fail"] = True + #pst.pestpp_options["overdue_giveup_fac"] = 1.0e+10 + #pst.pestpp_options["overdue_giveup_minutes"] = 0.25 + pst.write(os.path.join(new_d, "pest.pst")) + pst.control_data.noptmax = 2 + + pst.write(os.path.join(new_d, "pest.pst")) + m_d = os.path.join(model_d,"fr_timeout_master_freeze") + #num workers hard coded with conditional below + pyemu.os_utils.start_workers(new_d,exe_path,"pest.pst",num_workers=10,worker_root=model_d,master_dir=m_d) + #df = pyemu.helpers.parse_rmr_file(os.path.join(m_d,"pest.rmr")) + #print(df.action.to_list()) + oe = pd.read_csv(os.path.join(m_d,"pest.{0}.obs.csv".format(pst.control_data.noptmax)),index_col=0) + assert oe.shape[0] == 17 # hard coded to num reals + with open(os.path.join(m_d,"pest.rmr"),'r') as f: + for line in f: + if "timeout" in line.lower(): + raise Exception() + if line.strip().lower().endswith("agents connected"): + num = int(line.strip().split()[0]) + print(line.strip()) + assert num == 7 # hard coded above + + + +def ins_missing_e_test(): + import os + import shutil + import pyemu + t_d = os.path.join("tplins_test_1","test_missing_e") + if os.path.exists(t_d): + shutil.rmtree(t_d) + os.makedirs(t_d) + bd = os.getcwd() + os.chdir(t_d) + with open("model.output.bak",'w') as f: + f.write("12345-123\n") + pst = pyemu.helpers.pst_from_parnames_obsnames(["p1"],["o1"]) + pst.control_data.noptmax = 0 + with open("forward_run.py",'w') as f: + f.write("import shutil\n") + f.write("shutil.copy2('model.output.bak','model.output')\n") + pst.model_command = "python forward_run.py" + pst.write("test.pst") + os.chdir(bd) + try: + pyemu.os_utils.run("{0} test.pst".format(exe_path),cwd=t_d) + except: + pass + else: + raise Exception("should have failed") + + +def prep_ends(): + model_d = "mf6_freyberg" + base_d = os.path.join(model_d, "template") + new_d = os.path.join(model_d, "ends") + if os.path.exists(new_d): + shutil.rmtree(new_d) + os.makedirs(new_d) + skip = ["pst","csv","log","grb","hds","par","rei","lst","jcb","cov","rec","cbc"] + files = [f for f in os.listdir(base_d) if f.lower().split('.')[-1] not in skip] + print(files) + [shutil.copy2(os.path.join(base_d,f),os.path.join(new_d,f)) for f in files] + [shutil.copy2(os.path.join(base_d,f),os.path.join(new_d,f)) for f in ["ies_prior.jcb"]] -# pyemu.os_utils.run("mf6",cwd=new_d) -# pst = pyemu.Pst(os.path.join(base_d,"freyberg6_run_ies.pst")) -# pst.control_data.noptmax = 0 -# pst.pestpp_options = {} -# pst.pestpp_options["ies_par_en"] = "prior.jcb" - -# pst.write(os.path.join(new_d,"freyberg6_run_ies.pst"),version=2) -# pyemu.os_utils.run("pestpp-ies freyberg6_run_ies.pst",cwd=new_d) - -# build_and_draw_prior(new_d,num_reals=5000) -# pst.control_data.noptmax = -1 -# pst.write(os.path.join(new_d,"freyberg6_run_ies.pst"),version=2) -# m_d = os.path.join(model_d,"ends_master") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) - -# pyemu.os_utils.start_workers(new_d,"pestpp-ies","freyberg6_run_ies.pst",num_workers=15,worker_root=model_d,master_dir=m_d) - - -# def build_and_draw_prior(t_d="ends",num_reals=500): -# import flopy - -# sim = flopy.mf6.MFSimulation.load(sim_ws=t_d) -# m = sim.get_model("freyberg6") -# xgrid = m.modelgrid.xcellcenters -# ygrid = m.modelgrid.ycellcenters -# pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst")) -# par = pst.parameter_data -# static_par = par.loc[par.parnme.apply(lambda x: x[:3] in ["npf","sto"]),:].copy() -# static_par.loc[:, "i"] = static_par.parnme.apply(lambda x: int(x.split('_')[3])) -# static_par.loc[:, "j"] = static_par.parnme.apply(lambda x: int(x.split('_')[4])) -# static_par.loc[:, "x"] = static_par.apply(lambda x: xgrid[x.i,x.j],axis=1) -# static_par.loc[:, "y"] = static_par.apply(lambda x: ygrid[x.i, x.j], axis=1) -# static_par.loc[:,"pargp"] = static_par.parnme.apply(lambda x: "_".join(x.split('_')[:3])) - -# wel_par = par.loc[par.parnme.apply(lambda x: x.startswith("wel")),:].copy() -# wel_par.loc[:,"x"] = wel_par.parnme.apply(lambda x: int(x.split('_')[-1])) -# wel_par.loc[:,"y"] = 0.0 -# wel_par.loc[:,"pargp"] = wel_par.parnme.apply(lambda x: '_'.join(x.split('_')[:-1])) - -# rch_par = par.loc[par.parnme.str.startswith("rch"),:].copy() -# rch_par.loc[:,"x"] = rch_par.parnme.apply(lambda x: int(x.split('_')[-1])) -# rch_par.loc[:,"y"] = 0.0 - -# spatial_v = pyemu.geostats.ExpVario(contribution=1.0,a=1000.0) -# temporal_v = pyemu.geostats.ExpVario(contribution=1.0,a=3) -# spatial_gs = pyemu.geostats.GeoStruct(variograms=spatial_v) -# temporal_gs = pyemu.geostats.GeoStruct(variograms=temporal_v) - -# static_struct_dict = {spatial_gs:[]} -# sgrps = static_par.pargp.unique() -# sgrps.sort() -# for pargp in sgrps: -# static_struct_dict[spatial_gs].append(static_par.loc[static_par.pargp==pargp,["parnme","x","y","i","j"]]) -# temporal_struct_dict = {temporal_gs: [rch_par.loc[:, ["parnme", "x", "y"]]]} -# wgrps = wel_par.pargp.unique() -# wgrps.sort() -# for pargp in wgrps: -# temporal_struct_dict[temporal_gs].append(wel_par.loc[wel_par.pargp == pargp, ["parnme", "x", "y"]]) - -# struct_dict = static_struct_dict -# for k,v in temporal_struct_dict.items(): -# struct_dict[k] = v -# print(struct_dict) -# np.random.seed(pyemu.en.SEED) -# pe = pyemu.helpers.geostatistical_draws(pst,struct_dict=struct_dict,num_reals=num_reals) -# pe.to_binary(os.path.join(t_d,"prior.jcb")) - - -# def run(): -# model_d = "mf6_freyberg" -# t_d = os.path.join(model_d,"template") -# pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" -# pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, -# worker_root=model_d, port=4004) - -# def sweep_bin_test(): - -# model_d = "ies_10par_xsec" -# t_d = os.path.join(model_d,"template") -# m_d = os.path.join(model_d,"master_sweep_bin") -# if os.path.exists(m_d): -# shutil.rmtree(m_d) -# pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) -# pe = pyemu.ParameterEnsemble.from_uniform_draw(pst,num_reals=50)#.loc[:,pst.par_names[:2]] - -# pe.to_csv(os.path.join(t_d,"sweep_in.csv")) -# pe._df.index = pe.index.map(str) -# print(pe.index) -# pe.to_dense(os.path.join(t_d,"sweep_in.bin")) -# pst.pestpp_options["ies_par_en"] = "sweep_in.csv" -# pst.pestpp_options["sweep_forgive"] = True -# pst.pestpp_options["sweep_parameter_file"] = "sweep_in.bin" -# pst.control_data.noptmax = -1 -# pst.pestpp_options.pop("ies_num_reals",None) -# pst.write(os.path.join(t_d,"pest_forgive.pst")) -# pst.pestpp_options["sweep_output_file"] = "sweep_out.bin" -# pst.pestpp_options["sweep_chunk"] = 9 -# pst.pestpp_options["ies_include_base"] = False -# pst.write(os.path.join(t_d,"pest_forgive.pst")) -# m_d = os.path.join(model_d,"master_sweep_bin_base") -# pyemu.os_utils.start_workers(t_d, exe_path, "pest_forgive.pst", 10, master_dir=m_d, -# worker_root=model_d,port=port) -# df1 = pd.read_csv(os.path.join(m_d, "pest_forgive.0.obs.csv"),index_col=0) -# assert df1.shape[0] == pe.shape[0] -# m_d = os.path.join(model_d, "master_sweep_bin") -# pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-swp"), "pest_forgive.pst", 10, master_dir=m_d, -# worker_root=model_d, port=port) -# df2 = pyemu.Matrix.from_binary(os.path.join(m_d,"sweep_out.bin")).to_dataframe() -# print(df2) -# print(df1) -# assert df2.shape == df1.shape -# diff = (df1.values - df2.values) -# print(diff) -# print(diff.max()) -# print(np.abs(diff).max()) -# assert np.abs(diff).max() < 1e-7 - -def fail_test(): - raise Exception("fail please") + pyemu.os_utils.run("mf6",cwd=new_d) + pst = pyemu.Pst(os.path.join(base_d,"freyberg6_run_ies.pst")) + pst.control_data.noptmax = 0 + pst.pestpp_options = {} + pst.pestpp_options["ies_par_en"] = "prior.jcb" + + pst.write(os.path.join(new_d,"freyberg6_run_ies.pst"),version=2) + pyemu.os_utils.run("pestpp-ies freyberg6_run_ies.pst",cwd=new_d) + + build_and_draw_prior(new_d,num_reals=5000) + pst.control_data.noptmax = -1 + pst.write(os.path.join(new_d,"freyberg6_run_ies.pst"),version=2) + m_d = os.path.join(model_d,"ends_master") + if os.path.exists(m_d): + shutil.rmtree(m_d) + + pyemu.os_utils.start_workers(new_d,"pestpp-ies","freyberg6_run_ies.pst",num_workers=15,worker_root=model_d,master_dir=m_d) + + +def build_and_draw_prior(t_d="ends",num_reals=500): + import flopy + + sim = flopy.mf6.MFSimulation.load(sim_ws=t_d) + m = sim.get_model("freyberg6") + xgrid = m.modelgrid.xcellcenters + ygrid = m.modelgrid.ycellcenters + pst = pyemu.Pst(os.path.join(t_d,"freyberg6_run_ies.pst")) + par = pst.parameter_data + static_par = par.loc[par.parnme.apply(lambda x: x[:3] in ["npf","sto"]),:].copy() + static_par.loc[:, "i"] = static_par.parnme.apply(lambda x: int(x.split('_')[3])) + static_par.loc[:, "j"] = static_par.parnme.apply(lambda x: int(x.split('_')[4])) + static_par.loc[:, "x"] = static_par.apply(lambda x: xgrid[x.i,x.j],axis=1) + static_par.loc[:, "y"] = static_par.apply(lambda x: ygrid[x.i, x.j], axis=1) + static_par.loc[:,"pargp"] = static_par.parnme.apply(lambda x: "_".join(x.split('_')[:3])) + + wel_par = par.loc[par.parnme.apply(lambda x: x.startswith("wel")),:].copy() + wel_par.loc[:,"x"] = wel_par.parnme.apply(lambda x: int(x.split('_')[-1])) + wel_par.loc[:,"y"] = 0.0 + wel_par.loc[:,"pargp"] = wel_par.parnme.apply(lambda x: '_'.join(x.split('_')[:-1])) + + rch_par = par.loc[par.parnme.str.startswith("rch"),:].copy() + rch_par.loc[:,"x"] = rch_par.parnme.apply(lambda x: int(x.split('_')[-1])) + rch_par.loc[:,"y"] = 0.0 + + spatial_v = pyemu.geostats.ExpVario(contribution=1.0,a=1000.0) + temporal_v = pyemu.geostats.ExpVario(contribution=1.0,a=3) + spatial_gs = pyemu.geostats.GeoStruct(variograms=spatial_v) + temporal_gs = pyemu.geostats.GeoStruct(variograms=temporal_v) + + static_struct_dict = {spatial_gs:[]} + sgrps = static_par.pargp.unique() + sgrps.sort() + for pargp in sgrps: + static_struct_dict[spatial_gs].append(static_par.loc[static_par.pargp==pargp,["parnme","x","y","i","j"]]) + temporal_struct_dict = {temporal_gs: [rch_par.loc[:, ["parnme", "x", "y"]]]} + wgrps = wel_par.pargp.unique() + wgrps.sort() + for pargp in wgrps: + temporal_struct_dict[temporal_gs].append(wel_par.loc[wel_par.pargp == pargp, ["parnme", "x", "y"]]) + + struct_dict = static_struct_dict + for k,v in temporal_struct_dict.items(): + struct_dict[k] = v + print(struct_dict) + np.random.seed(pyemu.en.SEED) + pe = pyemu.helpers.geostatistical_draws(pst,struct_dict=struct_dict,num_reals=num_reals) + pe.to_binary(os.path.join(t_d,"prior.jcb")) + + +def run(): + model_d = "mf6_freyberg" + t_d = os.path.join(model_d,"template") + pst_name = "freyberg6_run_ies_glm_noloc_standard.pst" + pyemu.os_utils.start_workers(t_d, exe_path, pst_name, num_workers=15, + worker_root=model_d, port=4004) + +def sweep_bin_test(): + + model_d = "ies_10par_xsec" + t_d = os.path.join(model_d,"template") + m_d = os.path.join(model_d,"master_sweep_bin") + if os.path.exists(m_d): + shutil.rmtree(m_d) + pst = pyemu.Pst(os.path.join(t_d,"pest.pst")) + pe = pyemu.ParameterEnsemble.from_uniform_draw(pst,num_reals=50)#.loc[:,pst.par_names[:2]] + + pe.to_csv(os.path.join(t_d,"sweep_in.csv")) + pe._df.index = pe.index.map(str) + print(pe.index) + pe.to_dense(os.path.join(t_d,"sweep_in.bin")) + pst.pestpp_options["ies_par_en"] = "sweep_in.csv" + pst.pestpp_options["sweep_forgive"] = True + pst.pestpp_options["sweep_parameter_file"] = "sweep_in.bin" + pst.control_data.noptmax = -1 + pst.pestpp_options.pop("ies_num_reals",None) + pst.write(os.path.join(t_d,"pest_forgive.pst")) + pst.pestpp_options["sweep_output_file"] = "sweep_out.bin" + pst.pestpp_options["sweep_chunk"] = 9 + pst.pestpp_options["ies_include_base"] = False + pst.write(os.path.join(t_d,"pest_forgive.pst")) + m_d = os.path.join(model_d,"master_sweep_bin_base") + pyemu.os_utils.start_workers(t_d, exe_path, "pest_forgive.pst", 10, master_dir=m_d, + worker_root=model_d,port=port) + df1 = pd.read_csv(os.path.join(m_d, "pest_forgive.0.obs.csv"),index_col=0) + assert df1.shape[0] == pe.shape[0] + m_d = os.path.join(model_d, "master_sweep_bin") + pyemu.os_utils.start_workers(t_d, exe_path.replace("-ies", "-swp"), "pest_forgive.pst", 10, master_dir=m_d, + worker_root=model_d, port=port) + df2 = pyemu.Matrix.from_binary(os.path.join(m_d,"sweep_out.bin")).to_dataframe() + print(df2) + print(df1) + assert df2.shape == df1.shape + diff = (df1.values - df2.values) + print(diff) + print(diff.max()) + print(np.abs(diff).max()) + assert np.abs(diff).max() < 1e-7 + +#def fail_test(): +# raise Exception("fail please") if __name__ == "__main__": #run() - fail_test() #mf6_v5_ies_test() #prep_ends() - #sweep_bin_test() + sweep_bin_test() #mf6_v5_sen_test() #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-glm.exe"),os.path.join("..","bin","win","pestpp-glm.exe")) #shutil.copy2(os.path.join("..", "exe", "windows", "x64", "Debug", "pestpp-ies.exe"), diff --git a/src/libs/common/system_variables.cpp b/src/libs/common/system_variables.cpp index 00eac76b..7caa60b6 100644 --- a/src/libs/common/system_variables.cpp +++ b/src/libs/common/system_variables.cpp @@ -165,7 +165,7 @@ int start(string &cmd_string) // argv[icmd] = cmds[icmd].data(); //} //argv[cmds.size() + 1] = NULL; //last arg must be NULL - + arg_v.push_back(NULL); pid_t pid = fork(); if (pid == 0) diff --git a/src/libs/common/utilities.cpp b/src/libs/common/utilities.cpp index 56922790..3f716824 100644 --- a/src/libs/common/utilities.cpp +++ b/src/libs/common/utilities.cpp @@ -1134,7 +1134,7 @@ void read_dense_binary(const string& filename, vector& row_names, vector in.close(); in.open(filename.c_str(), ifstream::binary); - //in.seekg(first_record); + in.seekg(first_record); //resize the matrix now that we know big it should be matrix.resize(row_names.size(), col_names.size()); diff --git a/src/libs/pestpp_common/EnsembleMethodUtils.cpp b/src/libs/pestpp_common/EnsembleMethodUtils.cpp index ea171736..790f69b1 100644 --- a/src/libs/pestpp_common/EnsembleMethodUtils.cpp +++ b/src/libs/pestpp_common/EnsembleMethodUtils.cpp @@ -1374,13 +1374,13 @@ void UpgradeThread::ensemble_solution(const int iter, const int verbose_level,co Eigen::MatrixXd temp = parcov_inv.diagonal().matrix(); local_utils::save_mat(verbose_level, thread_id, iter, t_count, "parcov_inv", temp); } - if (act_obs_names.size() > 0) { //this works bc the mm solve doesnt pass these names... - ss.str(""); - ss << "solution scaling factor: " << scale << endl; - ss << "eigthresh: " << eigthresh << endl; - ss << "maxsing: " << maxsing << endl; - cout << ss.str() << endl; - } +// if (act_obs_names.size() > 0) { //this works bc the mm solve doesnt pass these names... +// ss.str(""); +// ss << "solution scaling factor: " << scale << endl; +// ss << "eigthresh: " << eigthresh << endl; +// ss << "maxsing: " << maxsing << endl; +// cout << ss.str() << endl; +// } } if (use_prior_scaling) diff --git a/src/libs/run_managers/abstract_base/model_interface.cpp b/src/libs/run_managers/abstract_base/model_interface.cpp index 7902b0b1..dfa314b4 100644 --- a/src/libs/run_managers/abstract_base/model_interface.cpp +++ b/src/libs/run_managers/abstract_base/model_interface.cpp @@ -274,7 +274,7 @@ void ThreadedTemplateProcess::work(int tid, vector& tpl_idx, Parameters par { if (tpl_idx.size() == 0) { - cout << "thread " << tid << " processed " << count << " template files" << endl; + //cout << "thread " << tid << " processed " << count << " template files" << endl; return; } i = tpl_idx[tpl_idx.size() - 1]; @@ -346,7 +346,8 @@ void ModelInterface::write_input_files(Parameters *pars_ptr) if (nnum_threads > tplfile_vec.size()) nnum_threads = tplfile_vec.size(); std::chrono::system_clock::time_point start_time = chrono::system_clock::now(); - cout << pest_utils::get_time_string() << " processing template files with " << nnum_threads << " threads..." << endl; + if (should_echo) + cout << pest_utils::get_time_string() << " processing template files with " << nnum_threads << " threads..." << endl; vector threads; vector exception_ptrs; Parameters pro_pars = *pars_ptr; //copy @@ -454,7 +455,8 @@ void ModelInterface::write_input_files(Parameters *pars_ptr) } pars_ptr->update_without_clear(pro_pars.get_keys(), pro_pars.get_data_vec(pro_pars.get_keys())); - cout << pest_utils::get_time_string() << " done, took " << pest_utils::get_duration_sec(start_time) << " seconds" << endl; + if (should_echo) + cout << pest_utils::get_time_string() << " done, took " << pest_utils::get_duration_sec(start_time) << " seconds" << endl; } void ModelInterface::read_output_files(Observations *obs) @@ -463,7 +465,8 @@ void ModelInterface::read_output_files(Observations *obs) if (nnum_threads > insfile_vec.size()) nnum_threads = insfile_vec.size(); std::chrono::system_clock::time_point start_time = chrono::system_clock::now(); - cout << pest_utils::get_time_string() << " processing instruction files with " << nnum_threads << " threads..." << endl; + if (should_echo) + cout << pest_utils::get_time_string() << " processing instruction files with " << nnum_threads << " threads..." << endl; vector threads; vector exception_ptrs; Observations temp_obs; @@ -600,7 +603,8 @@ void ModelInterface::read_output_files(Observations *obs) } t = temp_obs.get_keys(); obs->update(t, temp_obs.get_data_vec(t)); - cout << pest_utils::get_time_string() << " done, took " << pest_utils::get_duration_sec(start_time) << " seconds" << endl; + if (should_echo) + cout << pest_utils::get_time_string() << " done, took " << pest_utils::get_duration_sec(start_time) << " seconds" << endl; } @@ -661,7 +665,8 @@ void ModelInterface::run(pest_utils::thread_flag* terminate, pest_utils::thread_ write_input_files(pars_ptr); std::chrono::system_clock::time_point start_time = chrono::system_clock::now(); - cout << pest_utils::get_time_string() << " calling forward run command(s)" << endl; + if (should_echo) + cout << pest_utils::get_time_string() << " calling forward run command(s)" << endl; #ifdef OS_WIN //a flag to track if the run was terminated @@ -677,7 +682,8 @@ void ModelInterface::run(pest_utils::thread_flag* terminate, pest_utils::thread_ } for (auto &cmd_string : comline_vec) { - cout << pest_utils::get_time_string() << " calling forward run command: '" << cmd_string << "' " << endl; + if (should_echo) + cout << pest_utils::get_time_string() << " calling forward run command: '" << cmd_string << "' " << endl; //start the command PROCESS_INFORMATION pi; try @@ -694,7 +700,8 @@ void ModelInterface::run(pest_utils::thread_flag* terminate, pest_utils::thread_ throw PestError("could not add process to job object: " + cmd_string); } DWORD pid = pi.dwProcessId; - cout << "...pid: " << pid << endl; + if (should_echo) + cout << "...pid: " << pid << endl; DWORD exitcode; while (true) { @@ -717,7 +724,8 @@ void ModelInterface::run(pest_utils::thread_flag* terminate, pest_utils::thread_ //check for termination flag if (terminate->get()) { - std::cout << "received terminate signal" << std::endl; + if (should_echo) + std::cout << "received terminate signal" << std::endl; //try to kill the process bool success = (CloseHandle(job) != 0); @@ -745,10 +753,12 @@ void ModelInterface::run(pest_utils::thread_flag* terminate, pest_utils::thread_ bool term_break = false; for (auto &cmd_string : comline_vec) { - cout << pest_utils::get_time_string() << " calling forward run command: '" << cmd_string << "' " << endl; + if (should_echo) + cout << pest_utils::get_time_string() << " calling forward run command: '" << cmd_string << "' " << endl; //start the command int command_pid = start(cmd_string); - cout << "...pid: " << command_pid << endl; + if (should_echo) + cout << "...pid: " << command_pid << endl; while (true) { std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms)); @@ -770,7 +780,8 @@ void ModelInterface::run(pest_utils::thread_flag* terminate, pest_utils::thread_ //check for termination flag if (terminate->get()) { - std::cout << "received terminate signal" << std::endl; + if (should_echo) + std::cout << "received terminate signal" << std::endl; //try to kill the process errno = 0; int success = kill(-command_pid, SIGKILL); @@ -789,8 +800,8 @@ void ModelInterface::run(pest_utils::thread_flag* terminate, pest_utils::thread_ if (term_break) break; } #endif - - cout << pest_utils::get_time_string() << " foward run command(s) finished, took " << pest_utils::get_duration_sec(start_time) << " seconds" << endl; + if (should_echo) + cout << pest_utils::get_time_string() << " forward run command(s) finished, took " << pest_utils::get_duration_sec(start_time) << " seconds" << endl; if (term_break) return; diff --git a/src/libs/run_managers/abstract_base/model_interface.h b/src/libs/run_managers/abstract_base/model_interface.h index 93d64edf..7206272a 100644 --- a/src/libs/run_managers/abstract_base/model_interface.h +++ b/src/libs/run_managers/abstract_base/model_interface.h @@ -113,6 +113,7 @@ class ModelInterface{ void set_tpl_force_decimal(bool _flag) {tpl_force_decimal = _flag;} void set_num_threads(int _num_threads) { num_threads = _num_threads; } void set_sleep_ms(int _sleep_ms){sleep_ms = _sleep_ms;} + void set_should_echo(bool _should_echo){should_echo=_should_echo;} private: int num_threads; @@ -128,6 +129,7 @@ class ModelInterface{ bool fill_tpl_zeros; bool tpl_force_decimal; string additional_ins_delimiters; + bool should_echo; void write_input_files(Parameters *pars_ptr); void read_output_files(Observations *obs_ptr); diff --git a/src/libs/run_managers/serial/RunManagerSerial.cpp b/src/libs/run_managers/serial/RunManagerSerial.cpp index d5d7a2ce..1d9c5e0c 100644 --- a/src/libs/run_managers/serial/RunManagerSerial.cpp +++ b/src/libs/run_managers/serial/RunManagerSerial.cpp @@ -40,7 +40,7 @@ RunManagerSerial::RunManagerSerial(const vector _comline_vec, const vector _insfile_vec, const vector _outfile_vec, const string &stor_filename, const string &_run_dir, int _max_run_fail, bool fill_tpl_zeros, string additional_ins_delimiters, int _num_threads, - bool tpl_force_decimal) + bool tpl_force_decimal, bool should_echo) : RunManagerAbstract(_comline_vec, _tplfile_vec, _inpfile_vec, _insfile_vec, _outfile_vec, stor_filename, _max_run_fail), run_dir(_run_dir), mi(_tplfile_vec,_inpfile_vec,_insfile_vec,_outfile_vec, _comline_vec) @@ -50,6 +50,7 @@ RunManagerSerial::RunManagerSerial(const vector _comline_vec, mi.set_num_threads(_num_threads); mi.set_tpl_force_decimal(tpl_force_decimal); mi.set_sleep_ms(5); + mi.set_should_echo(should_echo); cout << " starting serial run manager ..." << endl << endl; mgr_type = RUN_MGR_TYPE::SERIAL; } diff --git a/src/libs/run_managers/serial/RunManagerSerial.h b/src/libs/run_managers/serial/RunManagerSerial.h index aca8189b..7d7e1554 100644 --- a/src/libs/run_managers/serial/RunManagerSerial.h +++ b/src/libs/run_managers/serial/RunManagerSerial.h @@ -31,13 +31,14 @@ class RunManagerSerial : public RunManagerAbstract const std::vector _insfile_vec, const std::vector _outfile_vec, const std::string &stor_filename, const std::string &run_dir, int _max_run_fail=1, bool fill_tpl_zeros=false, string additional_ins_delimiters="", int _num_threads=1, - bool tpl_force_decimal=false); + bool tpl_force_decimal=false, bool should_echo=true); virtual void run(); ~RunManagerSerial(void); private: ModelInterface mi; std::string run_dir; + void run_async(pest_utils::thread_flag* terminate, pest_utils::thread_flag* finished, exception_ptr& run_exception, Parameters* pars, Observations* obs); diff --git a/src/programs/pestpp-ies/pestpp-ies.cpp b/src/programs/pestpp-ies/pestpp-ies.cpp index 41257816..cce7cdb8 100644 --- a/src/programs/pestpp-ies/pestpp-ies.cpp +++ b/src/programs/pestpp-ies/pestpp-ies.cpp @@ -239,7 +239,8 @@ int main(int argc, char* argv[]) pest_scenario.get_pestpp_options().get_fill_tpl_zeros(), pest_scenario.get_pestpp_options().get_additional_ins_delimiters(), pest_scenario.get_pestpp_options().get_num_tpl_ins_threads(), - pest_scenario.get_pestpp_options().get_tpl_force_decimal()); + pest_scenario.get_pestpp_options().get_tpl_force_decimal(), + pest_scenario.get_pestpp_options().get_panther_echo()); } const ParamTransformSeq &base_trans_seq = pest_scenario.get_base_par_tran_seq(); From c6df7ed2935137e5dfcba98533d01498dc0b8829 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 30 Apr 2024 12:09:17 -0600 Subject: [PATCH 21/27] fix in sweep for binary read --- src/programs/sweep/pestpp-swp.vcxproj.user | 4 ++-- src/programs/sweep/sweep.cpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/programs/sweep/pestpp-swp.vcxproj.user b/src/programs/sweep/pestpp-swp.vcxproj.user index 249a734e..a0d40ce8 100644 --- a/src/programs/sweep/pestpp-swp.vcxproj.user +++ b/src/programs/sweep/pestpp-swp.vcxproj.user @@ -1,8 +1,8 @@  - 10par_xsec.pst - C:\Dev\pyemu\autotest\moouu\10par_xsec\template + pest_forgive.pst + C:\Dev\pestpp\benchmarks\ies_10par_xsec\master_sweep_bin WindowsLocalDebugger diff --git a/src/programs/sweep/sweep.cpp b/src/programs/sweep/sweep.cpp index 4740ed38..c96b9333 100644 --- a/src/programs/sweep/sweep.cpp +++ b/src/programs/sweep/sweep.cpp @@ -716,6 +716,8 @@ int main(int argc, char* argv[]) { cout << " --- dense binary file detected for par_csv" << endl; fout_rec << " --- dense binary file detected for par_csv" << endl; + par_stream.close(); + par_stream.open(par_csv_file, ifstream::binary); vector col_names; header_info = prepare_parameter_dense_binary(pest_scenario.get_ctl_parameters(),par_stream, pest_scenario.get_pestpp_options().get_sweep_forgive(),col_names); From 5c11b5b3005e4e819c38294458c7328b8f1100b0 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 2 May 2024 08:34:03 -0600 Subject: [PATCH 22/27] fix for msvc --- src/libs/pestpp_common/EnsembleMethodUtils.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libs/pestpp_common/EnsembleMethodUtils.cpp b/src/libs/pestpp_common/EnsembleMethodUtils.cpp index 790f69b1..9aa44d4d 100644 --- a/src/libs/pestpp_common/EnsembleMethodUtils.cpp +++ b/src/libs/pestpp_common/EnsembleMethodUtils.cpp @@ -5676,7 +5676,8 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing) //check to see if any explicit obs noise options are set bool reset_to_nonoise = true; - if (ppo->get_passed_args().find("IES_NO_NOISE") != ppo->get_passed_args().end()) + set passed = ppo->get_passed_args(); + if (passed.find("IES_NO_NOISE") != passed.end()) { reset_to_nonoise = false; } From 12c1a2c39c2a8b9a8b49589385388e3cbf597d17 Mon Sep 17 00:00:00 2001 From: jwhite Date: Thu, 2 May 2024 09:47:37 -0600 Subject: [PATCH 23/27] fix for default to no noise --- benchmarks/basic_tests.py | 4 +++- src/libs/pestpp_common/EnsembleMethodUtils.cpp | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/basic_tests.py b/benchmarks/basic_tests.py index a173203d..ba6b56fc 100644 --- a/benchmarks/basic_tests.py +++ b/benchmarks/basic_tests.py @@ -821,6 +821,7 @@ def ext_stdcol_test(): par.loc[pst.adj_par_names,"standard_deviation"] = (par.loc[pst.adj_par_names,"parubnd_trans"] - par.loc[pst.adj_par_names,"parlbnd_trans"]) / 4.0 #par.loc[pst.adj_par_names[0],"mean"] = par.loc[pst.adj_par_names[0],"parubnd"] pst.pestpp_options["ies_num_reals"] = 10 + pst.pestpp_options["ies_no_noise"] = False pst.control_data.noptmax = -1 pst.write(os.path.join(m_d,"pest_base.pst")) pyemu.os_utils.run("{0} pest_base.pst".format(exe_path),cwd=m_d) @@ -1488,8 +1489,9 @@ def sweep_bin_test(): #run() #mf6_v5_ies_test() #prep_ends() - sweep_bin_test() + #sweep_bin_test() #mf6_v5_sen_test() + ext_stdcol_test() #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-glm.exe"),os.path.join("..","bin","win","pestpp-glm.exe")) #shutil.copy2(os.path.join("..", "exe", "windows", "x64", "Debug", "pestpp-ies.exe"), # os.path.join("..", "bin", "win", "pestpp-ies.exe")) diff --git a/src/libs/pestpp_common/EnsembleMethodUtils.cpp b/src/libs/pestpp_common/EnsembleMethodUtils.cpp index 9aa44d4d..07fd013d 100644 --- a/src/libs/pestpp_common/EnsembleMethodUtils.cpp +++ b/src/libs/pestpp_common/EnsembleMethodUtils.cpp @@ -5685,12 +5685,15 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing) reset_to_nonoise = false; else if (!ppo->get_obscov_filename().empty()) reset_to_nonoise = false; + else if (ppo->get_ies_use_mda()) + reset_to_nonoise = false; else { map obs_std = pest_scenario.get_ext_file_double_map("observation data external", "standard_deviation"); if (obs_std.size() > 0) reset_to_nonoise = false; } + if (reset_to_nonoise) { ss.str(""); From a003f29e435d95b1648e7764b1dbfdf7c5695cd5 Mon Sep 17 00:00:00 2001 From: jwhite Date: Thu, 23 May 2024 16:14:44 -0600 Subject: [PATCH 24/27] some code cleanup, added nrelpar and phiredswh to control data keyword parsing --- .../pestpp_common/EnsembleMethodUtils.cpp | 950 ------------------ src/libs/pestpp_common/MOEA.cpp | 9 +- src/libs/pestpp_common/Pest.cpp | 2 +- src/libs/pestpp_common/pest_data_structs.cpp | 16 +- src/libs/pestpp_common/pest_data_structs.h | 2 +- 5 files changed, 22 insertions(+), 957 deletions(-) diff --git a/src/libs/pestpp_common/EnsembleMethodUtils.cpp b/src/libs/pestpp_common/EnsembleMethodUtils.cpp index 07fd013d..374afa38 100644 --- a/src/libs/pestpp_common/EnsembleMethodUtils.cpp +++ b/src/libs/pestpp_common/EnsembleMethodUtils.cpp @@ -1683,179 +1683,6 @@ void MmUpgradeThread::work(int thread_id, int iter, double cur_lam, bool use_glm UpgradeThread::ensemble_solution(iter,verbose_level,maxsing,thread_id,t_count, use_prior_scaling,use_approx,use_glm_form,cur_lam,eigthresh,par_resid,par_diff,Am,obs_resid, obs_diff,upgrade_1,obs_err,weights,parcov_inv,empty_obs_names,empty_par_names); -// Eigen::MatrixXd ivec, upgrade_1, s, s2, V, Ut, d_dash; -// -// //---------------------------------- -// //es-mda solution -// //---------------------------------- -// -// if (!use_glm_form) -// { -// if (true) -// { -// // Low rank Cee. Section 14.3.2 Evenson Book -// obs_err = obs_err.colwise() - obs_err.rowwise().mean(); -// obs_err = sqrt(cur_lam) * obs_err; -// Eigen::MatrixXd s0, V0, U0, s0_i; -// SVD_REDSVD rsvd; -// rsvd.solve_ip(obs_diff, s0, U0, V0, eigthresh, maxsing); -// s0_i = s0.asDiagonal().inverse(); -// Eigen::MatrixXd X0 = U0.transpose() * obs_err; -// X0 = s0_i * X0; -// Eigen::MatrixXd s1, V1, U1, s1_2, s1_2i; -// rsvd.solve_ip(X0, s1, U1, V1, 0, maxsing); -// -// s1_2 = s1.cwiseProduct(s1); -// s1_2i = (Eigen::VectorXd::Ones(s1_2.size()) + s1_2).asDiagonal().inverse(); -// Eigen::MatrixXd X1 = s0_i * U1; -// X1 = U0 * X1; -// -// Eigen::MatrixXd X4 = s1_2i * X1.transpose(); -// Eigen::MatrixXd X2 = X4 * obs_resid; -// Eigen::MatrixXd X3 = X1 * X2; -// -// X3 = obs_diff.transpose() * X3; -// upgrade_1 = -1 * par_diff * X3; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_1", upgrade_1); -// upgrade_1.transposeInPlace(); -// -// } -// else -// { -// // Use when ensemble size is larger than number of observation. -// // This is a good option when number of observation is small -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "obs_err", obs_err); -// obs_err = obs_err.colwise() - obs_err.rowwise().mean(); -// -// Eigen::MatrixXd s2_, s, V, U, cum_sum; -// SVD_REDSVD rsvd; -// Eigen::MatrixXd C; -// C = obs_diff + (sqrt(cur_lam) * obs_err); // curr_lam is the inflation factor -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "C", C); -// Eigen::VectorXd s2; -// -// -// rsvd.solve_ip(C, s, U, V, eigthresh, maxsing); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "s", s); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "U", U); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "V", V); -// s2 = s.cwiseProduct(s); -// s2_ = s2.asDiagonal().inverse(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "inv_s2_", s2_); -// -// -// Eigen::MatrixXd X1 = s2_ * U.transpose(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X1", X1); -// -// X1 = X1 * obs_resid; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X1_obs_resid", X1); -// -// X1 = U * X1; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "U_X1", X1); -// -// X1 = obs_diff.transpose() * X1; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "obs_diff_X1", X1); -// -// upgrade_1 = -1 * par_diff * X1; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_1", upgrade_1); -// upgrade_1.transposeInPlace(); -// } -// -// -// } -// -// -//// ---------------------------------- -//// glm solution -//// ---------------------------------- -// else -// { -// -// obs_resid = weights * obs_resid; -// obs_diff = scale * (weights * obs_diff); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "par_diff", par_diff); -// if (use_prior_scaling) -// par_diff = scale * parcov_inv * par_diff; -// else -// par_diff = scale * par_diff; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "scaled_par_diff", par_diff); -// SVD_REDSVD rsvd; -// rsvd.solve_ip(obs_diff, s, Ut, V, eigthresh, maxsing); -// -// Ut.transposeInPlace(); -// obs_diff.resize(0, 0); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "Ut", Ut); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "s", s); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "V", V); -// -// Eigen::MatrixXd s2 = s.cwiseProduct(s); -// -// ivec = ((Eigen::VectorXd::Ones(s2.size()) * (cur_lam + 1.0)) + s2).asDiagonal().inverse(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "ivec", ivec); -// -// Eigen::MatrixXd X1 = Ut * obs_resid; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X1", X1); -// -// Eigen::MatrixXd X2 = ivec * X1; -// X1.resize(0, 0); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X2", X2); -// -// Eigen::MatrixXd X3 = V * s.asDiagonal() * X2; -// X2.resize(0, 0); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X3", X3); -// upgrade_1 = -1.0 * par_diff * X3; -// -// if (use_prior_scaling) -// { -// //upgrade_1 = parcov_inv * upgrade_1; -// } -// -// upgrade_1.transposeInPlace(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_1", upgrade_1); -// X3.resize(0, 0); -// -// Eigen::MatrixXd upgrade_2; -// if ((!use_approx) && (iter > 1)) -// { -// if (use_prior_scaling) -// { -// par_resid = parcov_inv * par_resid; -// } -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "Am", Am); -// Eigen::MatrixXd x4 = Am.transpose() * par_resid; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X4", x4); -// -// par_resid.resize(0, 0); -// -// Eigen::MatrixXd x5 = Am * x4; -// x4.resize(0, 0); -// //Am.resize(0, 0); -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X5", x5); -// Eigen::MatrixXd x6 = par_diff.transpose() * x5; -// x5.resize(0, 0); -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X6", x6); -// Eigen::MatrixXd x7 = V * ivec * V.transpose() * x6; -// x6.resize(0, 0); -// -// if (use_prior_scaling) -// { -// upgrade_2 = -1.0 * parcov_inv * par_diff * x7; -// } -// else -// { -// upgrade_2 = -1.0 * (par_diff * x7); -// } -// x7.resize(0, 0); -// -// upgrade_1 = upgrade_1 + upgrade_2.transpose(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_2", upgrade_2); -// upgrade_2.resize(0, 0); -// -// } -// } //assuming that the fist row is the realization we are after... row_vec = upgrade_1.row(0); @@ -1900,493 +1727,6 @@ MmUpgradeThread::MmUpgradeThread(PerformanceLog* _performance_log, unordered_map } -//void CovLocalizationUpgradeThread::work(int thread_id, int iter, double cur_lam, bool use_glm_form, vector par_names, -// vector obs_names) -//{ -// -// //declare these helpers in here so they are thread safe... -// class local_utils -// { -// public: -// -// static void save_names(int verbose_level, int tid, int iter, int t_count, string prefix, vector& names) -// { -// if (verbose_level < 2) -// return; -// -// if (verbose_level < 3) -// return; -// stringstream ss; -// -// ss << "thread_" << tid << ".count_ " << t_count << ".iter_" << iter << "." << prefix << ".dat"; -// string fname = ss.str(); -// ofstream f(fname); -// if (!f.good()) -// cout << "error getting ofstream " << fname << endl; -// else -// { -// for (const auto& name : names) -// { -// f << name << endl; -// } -// -// } -// f.close(); -// return; -// } -// static Eigen::DiagonalMatrix get_matrix_from_map(vector& names, unordered_map& dmap) -// { -// Eigen::VectorXd vec(names.size()); -// int i = 0; -// for (auto name : names) -// { -// vec[i] = dmap.at(name); -// ++i; -// } -// Eigen::DiagonalMatrix m = vec.asDiagonal(); -// return m; -// } -// static Eigen::MatrixXd get_matrix_from_map(int num_reals, vector& names, unordered_map& emap) -// { -// Eigen::MatrixXd mat(num_reals, names.size()); -// mat.setZero(); -// -// for (int j = 0; j < names.size(); j++) -// { -// mat.col(j) = emap[names[j]]; -// } -// -// return mat; -// } -// static void save_mat(int verbose_level, int tid, int iter, int t_count, string prefix, Eigen::MatrixXd& mat) -// { -// if (verbose_level < 2) -// return; -// -// if (verbose_level < 3) -// return; -// //cout << "thread: " << tid << ", " << t_count << ", " << prefix << " rows:cols" << mat.rows() << ":" << mat.cols() << endl; -// stringstream ss; -// -// ss << "thread_" << tid << ".count_ " << t_count << ".iter_" << iter << "." << prefix << ".dat"; -// string fname = ss.str(); -// ofstream f(fname); -// if (!f.good()) -// cout << "error getting ofstream " << fname << endl; -// else -// { -// -// try -// { -// f << mat << endl; -// f.close(); -// } -// catch (...) -// { -// cout << "error saving matrix " << fname << endl; -// } -// } -// } -// }; -// -// stringstream ss; -// -// unique_lock ctrl_guard(ctrl_lock, defer_lock); -// int maxsing, num_reals, verbose_level, pcount = 0, t_count=0; -// double eigthresh; -// bool use_approx; -// bool use_prior_scaling; -// bool use_localizer = false; -// bool loc_by_obs = true; -// -// while (true) -// { -// if (ctrl_guard.try_lock()) -// { -// -// maxsing = pe_upgrade.get_pest_scenario_ptr()->get_svd_info().maxsing; -// eigthresh = pe_upgrade.get_pest_scenario_ptr()->get_svd_info().eigthresh; -// use_approx = pe_upgrade.get_pest_scenario_ptr()->get_pestpp_options().get_ies_use_approx(); -// use_prior_scaling = pe_upgrade.get_pest_scenario_ptr()->get_pestpp_options().get_ies_use_prior_scaling(); -// num_reals = pe_upgrade.shape().first; -// verbose_level = pe_upgrade.get_pest_scenario_ptr()->get_pestpp_options().get_ies_verbose_level(); -// ctrl_guard.unlock(); -// //if (pe_upgrade.get_pest_scenario_ptr()->get_pestpp_options().get_ies_localize_how()[0] == 'P') -// if (how == Localizer::How::PARAMETERS) -// loc_by_obs = false; -// else -// { -// throw runtime_error("Covariance localization only supporte for localization by parameters..."); -// } -// break; -// } -// } -// ofstream f_thread; -// local_utils::save_names(verbose_level,thread_id, iter, t_count,"act_obs_names",obs_names); -// local_utils::save_names(verbose_level,thread_id, iter, t_count,"act_par_names",par_names); -// if (verbose_level > 2) -// { -// ss.str(""); -// ss << "thread_" << thread_id << ".part_map.csv"; -// f_thread.open(ss.str()); -// ss.str(""); -// } -// Eigen::MatrixXd par_resid, par_diff, Am; -// Eigen::MatrixXd obs_resid, obs_diff, obs_err, loc; -// Eigen::DiagonalMatrix weights, parcov_inv; -// vector case_par_names, case_obs_names; -// string key; -// -// -// //these locks are used to control (thread-safe) access to the fast look up containers -// unique_lock next_guard(next_lock, defer_lock); -// unique_lock obs_diff_guard(obs_diff_lock, defer_lock); -// unique_lock obs_resid_guard(obs_resid_lock, defer_lock); -// unique_lock obs_err_guard(obs_err_lock, defer_lock); -// unique_lock par_diff_guard(par_diff_lock, defer_lock); -// unique_lock par_resid_guard(par_resid_lock, defer_lock); -// unique_lock loc_guard(loc_lock, defer_lock); -// unique_lock weight_guard(weight_lock, defer_lock); -// unique_lock parcov_guard(parcov_lock, defer_lock); -// unique_lock am_guard(am_lock, defer_lock); -// -// //reset all the solution parts -// par_resid.resize(0, 0); -// par_diff.resize(0, 0); -// obs_resid.resize(0, 0); -// obs_diff.resize(0, 0); -// obs_err.resize(0, 0); -// loc.resize(0, 0); -// Am.resize(0, 0); -// weights.resize(0); -// parcov_inv.resize(0); -// Am.resize(0, 0); -// -// -// -// obs_diff = local_utils::get_matrix_from_map(num_reals, obs_names, obs_diff_map); -// obs_resid = local_utils::get_matrix_from_map(num_reals, obs_names, obs_resid_map); -// obs_err = local_utils::get_matrix_from_map(num_reals, obs_names, obs_err_map); -// par_diff = local_utils::get_matrix_from_map(num_reals, par_names, par_diff_map); -// par_resid = local_utils::get_matrix_from_map(num_reals, par_names, par_resid_map); -// weights = local_utils::get_matrix_from_map(obs_names, weight_map); -// parcov_inv = local_utils::get_matrix_from_map(par_names, parcov_inv_map); -// -// if ((!use_approx) && (Am.rows() == 0)) -// { -// //Am = local_utils::get_matrix_from_map(num_reals, par_names, Am_map).transpose(); -// int am_cols = Am_map[par_names[0]].size(); -// Am.resize(par_names.size(), am_cols); -// Am.setZero(); -// -// for (int j = 0; j < par_names.size(); j++) -// { -// Am.row(j) = Am_map[par_names[j]]; -// } -// } -// -// par_diff.transposeInPlace(); -// obs_diff.transposeInPlace(); -// obs_resid.transposeInPlace(); -// par_resid.transposeInPlace(); -// obs_err.transposeInPlace(); -// -// -// //container to quickly look indices -// map par2col_map; -// for (int i = 0; i < par_names.size(); i++) -// par2col_map[par_names[i]] = i; -// -// map obs2row_map; -// for (int i = 0; i < obs_names.size(); i++) -// obs2row_map[obs_names[i]] = i; -// -// -// //form the scaled obs resid matrix -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "obs_resid", obs_resid); -// //Eigen::MatrixXd scaled_residual = weights * obs_resid; -// -// //form the (optionally) scaled par resid matrix -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "par_resid", par_resid); -// -// ss.str(""); -// double scale = (1.0 / (sqrt(double(num_reals - 1)))); -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "obs_diff", obs_diff); -// -// //calculate some full solution components first... -// -// Eigen::MatrixXd ivec, upgrade_1, s, s2, V, Ut, t; -// Eigen::MatrixXd upgrade_2; -// Eigen::VectorXd loc_vec; -// -// upgrade_2.resize(num_reals, pe_upgrade.shape().second); -// upgrade_2.setZero(); -// -// if (!use_glm_form) -// { -// if (true) -// { -// // Low rank Cee. Section 14.3.2 Evenson Book -// obs_err = obs_err.colwise() - obs_err.rowwise().mean(); -// obs_err = sqrt(cur_lam) * obs_err; -// Eigen::MatrixXd s0, V0, U0, s0_i; -// SVD_REDSVD rsvd; -// rsvd.solve_ip(obs_diff, s0, U0, V0, eigthresh, maxsing); -// s0_i = s0.asDiagonal().inverse(); -// Eigen::MatrixXd X0 = U0.transpose() * obs_err; -// X0 = s0_i * X0; -// Eigen::MatrixXd s1, V1, U1, s1_2, s1_2i; -// rsvd.solve_ip(X0, s1, U1, V1, 0, maxsing); -// -// s1_2 = s1.cwiseProduct(s1); -// s1_2i = (Eigen::VectorXd::Ones(s1_2.size()) + s1_2).asDiagonal().inverse(); -// Eigen::MatrixXd X1 = s0_i * U1; -// X1 = U0 * X1; -// -// Eigen::MatrixXd X4 = s1_2i * X1.transpose(); -// Eigen::MatrixXd X2 = X4 * obs_resid; -// Eigen::MatrixXd X3 = X1 * X2; -// -// X3 = obs_diff.transpose() * X3; -// //upgrade_1 = -1 * par_diff * X3; -// //local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_1", upgrade_1); -// //upgrade_1.transposeInPlace(); -// t = obs_diff * X3; -// Ut.resize(0, 0); -// obs_diff.resize(0, 0); -// X3.resize(0,0); -// X2.resize(0,0); -// X4.resize(0,0); -// X1.resize(0,0); -// -// } -// else { -// obs_diff = scale * obs_diff;// (H-Hm)/sqrt(N-1) -// par_diff = scale * par_diff;// (K-Km)/sqrt(N-1) -// obs_err = scale * obs_err; // (E-Em)/sqrt(N-1) -// -// SVD_REDSVD rsvd; -// Eigen::MatrixXd C = obs_diff + (cur_lam * obs_err); // curr_lam is the inflation factor -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "C", C); -// rsvd.solve_ip(C, s, Ut, V, eigthresh, maxsing); -// Ut.transposeInPlace(); -// V.resize(0, 0); -// C.resize(0, 0); -// obs_err.resize(0, 0); -// -// // s2 = s.asDiagonal().inverse(); -// s2 = s.cwiseProduct(s).asDiagonal().inverse(); -// for (int i = 0; i < s.size(); i++) { -// if (s(i) < 1e-50) { -// s2(i, i) = 0; -// } -// } -// -// t = obs_diff.transpose() * Ut.transpose() * s2 * Ut; -// Ut.resize(0, 0); -// obs_diff.resize(0, 0); -// } -// } -// -// -// //---------------------------------- -// //glm solution -// //---------------------------------- -// else -// { -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "obs_diff", obs_diff); -// obs_diff = scale * (weights * obs_diff); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "scaled_obs_diff", obs_diff); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "par_diff", par_diff); -// if (verbose_level > 1) { -// if (parcov_inv.size() < 10000) { -// Eigen::MatrixXd temp = parcov_inv.toDenseMatrix(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "parcov_inv", temp); -// } -// ss.str(""); -// ss << "solution scaling factor: " << scale; -// cout << ss.str() << endl; -// } -// if (use_prior_scaling) -// par_diff = scale * parcov_inv * par_diff; -// else -// par_diff = scale * par_diff; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "scaled_par_diff", par_diff); -// SVD_REDSVD rsvd; -// rsvd.solve_ip(obs_diff, s, Ut, V, eigthresh, maxsing); -// -// Ut.transposeInPlace(); -// obs_diff.resize(0, 0); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "Ut", Ut); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "s", s); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "V", V); -// -// Eigen::MatrixXd s2 = s.cwiseProduct(s); -// -// ivec = ((Eigen::VectorXd::Ones(s2.size()) * (cur_lam + 1.0)) + s2).asDiagonal().inverse(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "ivec", ivec); -// -// obs_resid = weights * obs_resid; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "scaled_obs_resid", obs_resid); -// t = V * s.asDiagonal() * ivec * Ut; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "t", t); -// -// if ((!use_approx) && (iter > 1)) -// { -// if (use_prior_scaling) -// { -// par_resid = parcov_inv * par_resid; -// } -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "Am", Am); -// Eigen::MatrixXd x4 = Am.transpose() * par_resid; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X4", x4); -// -// par_resid.resize(0, 0); -// -// Eigen::MatrixXd x5 = Am * x4; -// x4.resize(0, 0); -// Am.resize(0, 0); -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X5", x5); -// Eigen::MatrixXd x6 = par_diff.transpose() * x5; -// x5.resize(0, 0); -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X6", x6); -// Eigen::MatrixXd x7 = V * ivec * V.transpose() * x6; -// x6.resize(0, 0); -// -// if (use_prior_scaling) -// { -// upgrade_2 = -1.0 * parcov_inv * par_diff * x7; -// } -// else -// { -// upgrade_2 = -1.0 * (par_diff * x7); -// } -// x7.resize(0, 0); -// -// //add upgrade_2 piece -// //upgrade_1 = upgrade_1 + upgrade_2.transpose(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_2", upgrade_2); -// //upgrade_2.resize(0, 0); -// } -// } -// -// //This is the main thread loop - it continues until all upgrade pieces have been completed -// map loc_map; -// //solve for each case par_name -// Eigen::VectorXd pt = parcov_inv.diagonal(); -// string name; -// while (true) -// { -// //clear the pieces used last time... -// key = ""; -// loc_map.clear(); -// -// //get new pieces... -// while (true) -// { -// if ((key != "") && (loc_map.size() > 0)) -// break; -// if (next_guard.try_lock()) -// { -// //if all the pieces have been completed, return -// if (count == keys.size()) -// { -// if (verbose_level > 3) -// { -// cout << "upgrade thread: " << thread_id << " processed " << pcount << " upgrade parts" << endl; -// } -// if (f_thread.good()) -// f_thread.close(); -// return; -// } -// key = keys[count]; -// pair, vector> p = cases.at(key); -// //we can potentially optimize the speed of this loc type by changing how many pars are -// //passed in each "case" so herein, we support a generic number of pars per case... -// case_par_names = p.second; -// //In this solution, we ignore case obs names since we are using the full set of obs for the solution... -// case_obs_names = p.first; -// -// if (count % 1000 == 0) -// { -// ss.str(""); -// ss << "upgrade thread progress: " << count << " of " << total << " parts done"; -// if (verbose_level > 3) -// cout << ss.str() << endl; -// performance_log->log_event(ss.str()); -// } -// count++; -// t_count = count; -// pcount++; -// next_guard.unlock(); -// -// } -// //get access to the localizer -// if ((key != "") && (loc_map.size() == 0) && (loc_guard.try_lock())) -// { -// //get the nobs-length localizing vector for each case par name -// for (auto& par_name : case_par_names) -// { -// loc_map[par_name] = localizer.get_obs_hadamard_vector(par_name, obs_names); -// } -// loc_guard.unlock(); -// } -// } -// -// if (verbose_level > 2) -// { -// f_thread << t_count << "," << iter; -// for (auto name : case_par_names) -// f_thread << "," << name; -// for (auto name : case_obs_names) -// f_thread << "," << name; -// f_thread << endl; -// } -// upgrade_1.resize(num_reals,case_par_names.size()); -// upgrade_1.setZero(); -// -// for (int i = 0; i < case_par_names.size(); i++) -// { -// name = case_par_names[i]; -// loc_vec = loc_map[name]; -// Eigen::VectorXd par_vec = par_diff.row(par2col_map[name]) * t; -// //apply the localizer -// par_vec = par_vec.cwiseProduct(loc_vec); -// if (!use_glm_form) -// par_vec = -1.0 * par_vec.transpose() * obs_resid; -// else -// { -// par_vec = -1.0 * par_vec.transpose() * obs_resid; -// //if (use_prior_scaling) -// // par_vec *= pt[i]; -// } -// upgrade_1.col(i) += par_vec.transpose(); -// //add the par change part for the full glm solution -// if ((use_glm_form) && (!use_approx) && (iter > 1)) -// { -// //update_2 is transposed relative to upgrade_1 -// upgrade_1.col(i) += upgrade_2.row(par2col_map[name]); -// } -// -// } -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_1", upgrade_1); -// -// //put this piece of the upgrade vector in -// unique_lock put_guard(put_lock, defer_lock); -// while (true) -// { -// if (put_guard.try_lock()) -// { -// pe_upgrade.add_2_cols_ip(case_par_names, upgrade_1); -// put_guard.unlock(); -// break; -// } -// } -// } -//} void LocalAnalysisUpgradeThread::work(int thread_id, int iter, double cur_lam, bool use_glm_form, @@ -2590,19 +1930,6 @@ void LocalAnalysisUpgradeThread::work(int thread_id, int iter, double cur_lam, b } } -// local_utils::save_names(verbose_level,thread_id, iter, t_count,"act_obs_names",obs_names); -// local_utils::save_names(verbose_level,thread_id, iter, t_count,"act_par_names",par_names); -// -// if (verbose_level > 2) -// { -// -// f_thread << t_count << "," << iter; -// for (auto name : par_names) -// f_thread << "," << name; -// for (auto name : obs_names) -// f_thread << "," << name; -// f_thread << endl; -// } //reset all the solution parts par_resid.resize(0, 0); @@ -2673,227 +2000,6 @@ void LocalAnalysisUpgradeThread::work(int thread_id, int iter, double cur_lam, b parcov_inv, obs_names,par_names); - -// par_diff.transposeInPlace(); -// obs_diff.transposeInPlace(); -// obs_resid.transposeInPlace(); -// par_resid.transposeInPlace(); -// obs_err.transposeInPlace(); -// -// //form the scaled obs resid matrix -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "obs_resid", obs_resid); -// //Eigen::MatrixXd scaled_residual = weights * obs_resid; -// -// //form the (optionally) scaled par resid matrix -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "par_resid", par_resid); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "par_diff", par_diff); -// -// stringstream ss; -// -// double scale = (1.0 / (sqrt(double(num_reals - 1)))); -// -// if (verbose_level > 1) { -// ss.str(""); -// ss << "solution scaling factor: " << scale; -// cout << ss.str() << endl; -// if (parcov_inv.size() < 10000) -// { -// Eigen::MatrixXd temp = parcov_inv.toDenseMatrix(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "parcov_inv", temp); -// } -// } -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "obs_diff", obs_diff); -// //apply the localizer here... -// if (use_localizer) -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "loc", loc); -// if (use_localizer) -// { -// if (loc_by_obs) -// par_diff = par_diff.cwiseProduct(loc); -// else -// obs_diff = obs_diff.cwiseProduct(loc); -// } -// -// //Eigen::MatrixXd upgrade_1; -//// ensemble_solution(iter,verbose_level,maxsing,thread_id,t_count, use_prior_scaling,use_approx,use_glm_form,cur_lam,eigthresh,par_resid,par_diff,Am,obs_resid, -//// obs_diff,upgrade_1,obs_err,weights,parcov_inv); -// -// -// -// //---------------------------------- -// //es-mda solution -// //---------------------------------- -// -// if (!use_glm_form) -// { -// if (true) -// { -// // Low rank Cee. Section 14.3.2 Evenson Book -// obs_err = obs_err.colwise() - obs_err.rowwise().mean(); -// obs_err = sqrt(cur_lam) * obs_err; -// Eigen::MatrixXd s0, V0, U0, s0_i; -// SVD_REDSVD rsvd; -// rsvd.solve_ip(obs_diff, s0, U0, V0, eigthresh, maxsing); -// s0_i = s0.asDiagonal().inverse(); -// Eigen::MatrixXd X0 = U0.transpose() * obs_err; -// X0 = s0_i * X0; -// Eigen::MatrixXd s1, V1, U1, s1_2, s1_2i; -// rsvd.solve_ip(X0, s1, U1, V1, 0, maxsing); -// -// s1_2 = s1.cwiseProduct(s1); -// s1_2i = (Eigen::VectorXd::Ones(s1_2.size()) + s1_2).asDiagonal().inverse(); -// Eigen::MatrixXd X1 = s0_i * U1; -// X1 = U0 * X1; -// -// Eigen::MatrixXd X4 = s1_2i * X1.transpose(); -// Eigen::MatrixXd X2 = X4 * obs_resid; -// Eigen::MatrixXd X3 = X1 * X2; -// -// X3 = obs_diff.transpose() * X3; -// upgrade_1 = -1 * par_diff * X3; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_1", upgrade_1); -// upgrade_1.transposeInPlace(); -// -// } -// else -// { -// // Use when ensemble size is larger than number of observation. -// // This is a good option when number of observation is small -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "obs_err", obs_err); -// obs_err = obs_err.colwise() - obs_err.rowwise().mean(); -// -// Eigen::MatrixXd s2_, s, V, U, cum_sum; -// SVD_REDSVD rsvd; -// Eigen::MatrixXd C; -// C = obs_diff + (sqrt(cur_lam) * obs_err); // curr_lam is the inflation factor -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "C", C); -// Eigen::VectorXd s2; -// -// -// rsvd.solve_ip(C, s, U, V, eigthresh, maxsing); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "s", s); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "U", U); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "V", V); -// s2 = s.cwiseProduct(s); -// s2_ = s2.asDiagonal().inverse(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "inv_s2_", s2_); -// -// -// Eigen::MatrixXd X1 = s2_ * U.transpose(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X1", X1); -// -// X1 = X1 * obs_resid; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X1_obs_resid", X1); -// -// X1 = U * X1; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "U_X1", X1); -// -// X1 = obs_diff.transpose() * X1; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "obs_diff_X1", X1); -// -// upgrade_1 = -1 * par_diff * X1; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_1", upgrade_1); -// upgrade_1.transposeInPlace(); -// } -// -// -// } -// -// -//// ---------------------------------- -//// glm solution -//// ---------------------------------- -// else -// { -// -// obs_resid = weights * obs_resid; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "scaled_obs_resid", par_diff); -// obs_diff = scale * (weights * obs_diff); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "scaled_obs_diff", par_diff); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "par_diff", par_diff); -// if (use_prior_scaling) -// par_diff = scale * parcov_inv * par_diff; -// else -// par_diff = scale * par_diff; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "scaled_par_diff", par_diff); -// SVD_REDSVD rsvd; -// rsvd.solve_ip(obs_diff, s, Ut, V, eigthresh, maxsing); -// -// Ut.transposeInPlace(); -// obs_diff.resize(0, 0); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "Ut", Ut); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "s", s); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "V", V); -// -// Eigen::MatrixXd s2 = s.cwiseProduct(s); -// -// ivec = ((Eigen::VectorXd::Ones(s2.size()) * (cur_lam + 1.0)) + s2).asDiagonal().inverse(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "ivec", ivec); -// -// Eigen::MatrixXd X1 = Ut * obs_resid; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X1", X1); -// -// Eigen::MatrixXd X2 = ivec * X1; -// X1.resize(0, 0); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X2", X2); -// -// Eigen::MatrixXd X3 = V * s.asDiagonal() * X2; -// X2.resize(0, 0); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X3", X3); -// upgrade_1 = -1.0 * par_diff * X3; -// -// if (use_prior_scaling) -// { -// //upgrade_1 = parcov_inv * upgrade_1; -// } -// -// upgrade_1.transposeInPlace(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_1", upgrade_1); -// X3.resize(0, 0); -// -// Eigen::MatrixXd upgrade_2; -// if ((!use_approx) && (iter > 1)) -// { -// if (use_prior_scaling) -// { -// par_resid = parcov_inv * par_resid; -// } -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "Am", Am); -// Eigen::MatrixXd x4 = Am.transpose() * par_resid; -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X4", x4); -// -// par_resid.resize(0, 0); -// -// Eigen::MatrixXd x5 = Am * x4; -// x4.resize(0, 0); -// Am.resize(0, 0); -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X5", x5); -// Eigen::MatrixXd x6 = par_diff.transpose() * x5; -// x5.resize(0, 0); -// -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "X6", x6); -// Eigen::MatrixXd x7 = V * ivec * V.transpose() * x6; -// x6.resize(0, 0); -// -// if (use_prior_scaling) -// { -// upgrade_2 = -1.0 * parcov_inv * par_diff * x7; -// } -// else -// { -// upgrade_2 = -1.0 * (par_diff * x7); -// } -// x7.resize(0, 0); -// -// upgrade_1 = upgrade_1 + upgrade_2.transpose(); -// local_utils::save_mat(verbose_level, thread_id, iter, t_count, "upgrade_2", upgrade_2); -// upgrade_2.resize(0, 0); -// -// } -// } while (true) { @@ -3710,62 +2816,6 @@ void L2PhiHandler::report(bool echo, bool group_report) -//void PhiHandler::report(bool echo) -//{ -// ofstream &f = file_manager->rec_ofstream(); -// f << get_summary_header(); -// if (echo) -// cout << get_summary_header(); -// string s = get_summary_string(PhiHandler::phiType::COMPOSITE); -// f << s; -// if (echo) -// cout << s; -// s = get_summary_string(PhiHandler::phiType::MEAS); -// f << s; -// if (echo) -// cout << s; -// if (org_reg_factor != 0.0) -// { -// s = get_summary_string(PhiHandler::phiType::REGUL); -// f << s; -// if (echo) -// cout << s; -// } -// s = get_summary_string(PhiHandler::phiType::ACTUAL); -// f << s; -// if (echo) -// cout << s; -// if (org_reg_factor != 0.0) -// { -// if (*reg_factor == 0.0) -// { -// f << " (note: reg_factor is zero; regularization phi reported but not used)" << endl; -// if (echo) -// cout << " (note: reg_factor is zero; regularization phi reported but not used)" << endl; -// } -// else -// { -// f << " current reg_factor: " << *reg_factor << endl; -// if (echo) -// cout << " current reg_factor: " << *reg_factor << endl; -// } -// if (*reg_factor != 0.0) -// { -// -// f << " note: regularization phi reported above does not " << endl; -// f << " include the effects of reg_factor, " << endl; -// f << " but composite phi does." << endl; -// if (echo) -// { -// cout << " note: regularization phi reported above does not " << endl; -// cout << " include the effects of reg_factor, " << endl; -// cout << " but composite phi does." << endl; -// } -// } -// } -// f << endl << endl; -// f.flush(); -//} void L2PhiHandler::write(int iter_num, int total_runs, bool write_group) { diff --git a/src/libs/pestpp_common/MOEA.cpp b/src/libs/pestpp_common/MOEA.cpp index 9db7a5b2..61847e5b 100644 --- a/src/libs/pestpp_common/MOEA.cpp +++ b/src/libs/pestpp_common/MOEA.cpp @@ -3389,17 +3389,20 @@ vector MOEA::get_pso_gbest_solutions(int num_reals, ParameterEnsemble& _ DomPair dompair = objectives.get_nsga2_pareto_dominance(-999, _op, _dp, &constraints, false); vector nondom_solutions = dompair.first; vector gbest_solutions; + int num_objs = pi_obj_names.size()+obs_obj_names.size(); + //if no non dom solutions, then use the dominated ones... if (nondom_solutions.size() == 0) { ss.str(""); - ss << "WARNING: no nondom solutions for pst gbest calculation, using dominated solutions" << endl; + ss << "WARNING: no nondom solutions for pso gbest calculation, using dominated solutions" << endl; nondom_solutions = dompair.second; } - else if (nondom_solutions.size() == 1) + //todo: should we warn for nondom > 1 and objs == 1? + else if ((nondom_solutions.size() == 1) && (num_objs > 1)) { ss.str(""); - ss << "WARNING: only one nondom solution for pst gbest calculation" << endl; + ss << "WARNING: only one nondom solution for pso gbest calculation" << endl; file_manager.rec_ofstream() << ss.str(); cout << ss.str(); for (int i = 0; i < num_reals; i++) diff --git a/src/libs/pestpp_common/Pest.cpp b/src/libs/pestpp_common/Pest.cpp index a6aaf3e9..6c94eefe 100644 --- a/src/libs/pestpp_common/Pest.cpp +++ b/src/libs/pestpp_common/Pest.cpp @@ -1141,7 +1141,7 @@ int Pest::process_ctl_file(ifstream& fin, string _pst_filename, ofstream& f_rec) //try to use this as a control data arg if (stat == PestppOptions::ARG_STATUS::ARG_NOTFOUND) { - stat = control_info.assign_value_by_key(kv.first,kv.second); + stat = control_info.assign_value_by_key(kv.first,kv.second,f_rec); check_report_assignment(f_rec, stat, kv.first, kv.second); } diff --git a/src/libs/pestpp_common/pest_data_structs.cpp b/src/libs/pestpp_common/pest_data_structs.cpp index ef28e356..11de6594 100644 --- a/src/libs/pestpp_common/pest_data_structs.cpp +++ b/src/libs/pestpp_common/pest_data_structs.cpp @@ -2251,7 +2251,7 @@ ostream& operator<< (ostream &os, const SVDInfo& val) return os; } -PestppOptions::ARG_STATUS ControlInfo::assign_value_by_key(const string key, const string org_value) +PestppOptions::ARG_STATUS ControlInfo::assign_value_by_key(const string key, const string org_value, ofstream& f_rec) { /*enum PestMode { ESTIMATION, REGUL, PARETO, UNKNOWN }; double phiredswh; @@ -2266,6 +2266,7 @@ PestppOptions::ARG_STATUS ControlInfo::assign_value_by_key(const string key, con int noptswitch; double splitswh; PestMode pestmode;*/ + stringstream ss; set valid_args{"RSTFLE","PESTMODE","NPAR","NOBS","NPARGP","NPRIOR","NOBSGP","MAXCOMPDIM","NTPLFLE","NINSFLE", "PRECIS","DPOINT","NUMCOM","JACFILE","MESSFILE","OBSREREF","RLAMBDA1","RLAMFAC","PHIREDLAM", "PHIRATSUF","NUMLAM","JACUPDATE","LAMFORGIVE","DERFORGIVE","RELPARMAX","FACPARMAX", @@ -2298,6 +2299,14 @@ PestppOptions::ARG_STATUS ControlInfo::assign_value_by_key(const string key, con convert_ip(value, noptswitch); else if (key == "SPLITSWH") convert_ip(value, splitswh); + else if (key == "PHIREDSWH") + { + convert_ip(value,phiredswh); + } + else if (key == "NRELPAR") + { + convert_ip(value, nrelpar); + } else if (key == "PESTMODE") { if (value == "ESTIMATION") @@ -2311,7 +2320,10 @@ PestppOptions::ARG_STATUS ControlInfo::assign_value_by_key(const string key, con } else if (valid_args.find(key) != valid_args.end()) { - //a valid but unused ctl data arg + ss.str(""); + ss << "WARNING: unused 'control data keyword' option,value:" << key << "," << value; + f_rec << ss.str(); + cout << ss.str(); } else return PestppOptions::ARG_STATUS::ARG_NOTFOUND; diff --git a/src/libs/pestpp_common/pest_data_structs.h b/src/libs/pestpp_common/pest_data_structs.h index a569a323..0236ef2e 100644 --- a/src/libs/pestpp_common/pest_data_structs.h +++ b/src/libs/pestpp_common/pest_data_structs.h @@ -879,7 +879,7 @@ class ControlInfo { int noptswitch; double splitswh; PestMode pestmode; - PestppOptions::ARG_STATUS assign_value_by_key(const string key, const string org_value); + PestppOptions::ARG_STATUS assign_value_by_key(const string key, const string org_value, ofstream& f_rec); ControlInfo() { ; } void set_defaults(); From 014a6eec5ddd82c8777524ad0355402b58fa77e5 Mon Sep 17 00:00:00 2001 From: jwhite Date: Thu, 30 May 2024 20:57:56 -0600 Subject: [PATCH 25/27] revised drop conflicts user feedback --- benchmarks/basic_tests.py | 4 ++-- src/libs/pestpp_common/EnsembleMethodUtils.cpp | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/benchmarks/basic_tests.py b/benchmarks/basic_tests.py index ba6b56fc..0ae729ea 100644 --- a/benchmarks/basic_tests.py +++ b/benchmarks/basic_tests.py @@ -1491,7 +1491,7 @@ def sweep_bin_test(): #prep_ends() #sweep_bin_test() #mf6_v5_sen_test() - ext_stdcol_test() + #ext_stdcol_test() #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-glm.exe"),os.path.join("..","bin","win","pestpp-glm.exe")) #shutil.copy2(os.path.join("..", "exe", "windows", "x64", "Debug", "pestpp-ies.exe"), # os.path.join("..", "bin", "win", "pestpp-ies.exe")) @@ -1529,7 +1529,7 @@ def sweep_bin_test(): #shutil.copy2(os.path.join("..","exe","windows","x64","Debug","pestpp-ies.exe"),os.path.join("..","bin","win","pestpp-ies.exe")) #tplins1_test() - #fr_timeout_test() + fr_timeout_test() #mf6_v5_ies_test() #mf6_v5_sen_test() diff --git a/src/libs/pestpp_common/EnsembleMethodUtils.cpp b/src/libs/pestpp_common/EnsembleMethodUtils.cpp index 374afa38..aa260072 100644 --- a/src/libs/pestpp_common/EnsembleMethodUtils.cpp +++ b/src/libs/pestpp_common/EnsembleMethodUtils.cpp @@ -5225,13 +5225,16 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing) ss.str(""); ss << " WARNING: Prior-data conflict detected. Continuing with IES parameter" << endl; ss << " adjustment will likely result in parameter and forecast bias." << endl; - ss << " Consider using 'ies_drop_conflicts' as a quick fix."; + //ss << " Consider using 'ies_drop_conflicts' as a quick fix."; message(1, ss.str()); } else { - - //check that all obs are in conflict + ss.str(""); + ss << " WARNING: 'ies_drop_conflicts' is not an ideal approach. A better" << endl; + ss << " approach would be to dig in and figure out why these observations " << endl; + ss << " are in conflict." << endl; + //check that all obs are in conflict message(1, "dropping conflicted observations"); if (in_conflict.size() == act_obs_names.size()) { From 53555a623449b03f74aa95e762751ff74e233d37 Mon Sep 17 00:00:00 2001 From: jwhite Date: Fri, 7 Jun 2024 09:29:40 -0600 Subject: [PATCH 26/27] fix in opt for tied dv pars --- src/libs/pestpp_common/Pest.cpp | 521 ----------------------- src/libs/pestpp_common/sequential_lp.cpp | 45 +- 2 files changed, 37 insertions(+), 529 deletions(-) diff --git a/src/libs/pestpp_common/Pest.cpp b/src/libs/pestpp_common/Pest.cpp index 6c94eefe..738dc8eb 100644 --- a/src/libs/pestpp_common/Pest.cpp +++ b/src/libs/pestpp_common/Pest.cpp @@ -463,12 +463,6 @@ vector Pest::get_nonregul_obs() const return ret_val; } -//int Pest::process_ctl_file_old(ifstream &fin, string pst_filename) -//{ -// ofstream f_out("ctl_process.out"); -// return process_ctl_file_old(fin, pst_filename, f_out); -//} - int Pest::process_ctl_file(ifstream& fin, string _pst_filename) { ofstream f_out("ctl_process.out"); @@ -476,521 +470,6 @@ int Pest::process_ctl_file(ifstream& fin, string _pst_filename) } -//int Pest::process_ctl_file_old(ifstream &fin, string _pst_filename, ofstream &f_rec) -//{ -// string line; -// string line_upper; -// string section(""); -// vector tokens; -// int sec_begin_lnum, sec_lnum; -// double value; -// string name; -// string *trans_type; -// string prior_info_string; -// pair pi_name_group; -// int lnum; -// int num_par; -// int num_tpl_file; -// int dercom; -// int i_tpl_ins = 0; -// double phimlim; -// double phimaccept; -// double fracphim; -// double wfinit = 1.0; -// double wfmin = numeric_limits::min(); -// double wfmax = numeric_limits::max(); -// double wffac; -// double wftol; -// bool use_dynamic_reg = false; -// bool reg_adj_grp_weights = false; -// vector pestpp_input; -// regul_scheme_ptr = new DynamicRegularization(use_dynamic_reg); -// -// //dont change these text names - they are used in ParamTransformSeq -// TranTied *t_tied = new TranTied("PEST to model tied transformation"); -// TranOffset *t_offset = new TranOffset("PEST to model offset transformation"); -// TranScale *t_scale = new TranScale("PEST to model scale transformation"); -// TranLog10 *t_log = new TranLog10("PEST to model log transformation"); -// TranFixed *t_fixed = new TranFixed("PEST to model fixed transformation"); -// //TranNormalize *t_auto_norm = new TranNormalize("PEST auto-normalization transformation"); -// -// base_par_transform.push_back_ctl2model(t_scale); -// base_par_transform.push_back_ctl2model(t_offset); -// base_par_transform.push_back_ctl2active_ctl(t_tied); -// base_par_transform.push_back_ctl2active_ctl(t_fixed); -// base_par_transform.push_back_active_ctl2numeric(t_log); -// -// -// set tied_names; -// pst_filename = _pst_filename; -// -//#ifndef _DEBUG -// try { -//#endif -// prior_info_string = ""; -// for(lnum=1, sec_begin_lnum=1; getline(fin, line); ++ lnum) -// { -// strip_ip(line); -// line_upper = upper_cp(line); -// tokens.clear(); -// tokenize(line_upper, tokens); -// sec_lnum = lnum - sec_begin_lnum; -// -// if (lnum == 1) -// { -// if (tokens[0] != "PCF") -// { -// cout << "WARNING: fist line of control file should be 'PCF' not " << tokens[0] << endl; -// } -// } -// -// else if (tokens.empty()) -// { -// //skip blank line -// } -// else if (line[0] == '#') -// { -// -// } -// else if (line_upper.substr(0,2) == "++") -// { -// pestpp_input.push_back(line); -// } -// -// else if (line_upper[0] == '*') -// { -// section = upper_cp(strip_cp(line_upper, "both", " *\t\n")); -// sec_begin_lnum = lnum; -// } -// else if (section == "CONTROL DATA") -// { -// if (sec_lnum == 1) -// { -// if (tokens[1] == "REGULARIZATION" || tokens[1] == "REGULARISATION") -// { -// use_dynamic_reg = true; -// control_info.pestmode = ControlInfo::PestMode::REGUL; -// -// } -// else if (tokens[1] == "ESTIMATION") -// control_info.pestmode = ControlInfo::PestMode::ESTIMATION; -// else if (tokens[1] == "PARETO") -// control_info.pestmode = ControlInfo::PestMode::PARETO; -// } -// -// -// else if (sec_lnum == 2) -// { -// convert_ip(tokens[0], num_par); -// } -// else if (sec_lnum == 3) -// { -// convert_ip(tokens[0], num_tpl_file); -// if(tokens.size() >= 5) { -// convert_ip(tokens[4], control_info.numcom); -// } -// else { -// control_info.numcom = 0; -// } -// if(tokens.size() >= 6) { -// convert_ip(tokens[5], control_info.jacfile); -// } -// else { -// control_info.jacfile = 0; -// } -// } -// else if (sec_lnum == 5) -// { -// convert_ip(tokens[0], control_info.relparmax); -// convert_ip(tokens[1], control_info.facparmax); -// convert_ip(tokens[2], control_info.facorig); -// } -// else if (sec_lnum == 6) -// { -// // remove text arguements from the line as these can be specified out of order -// // and PEST++ does not use them -// set remove_tags = { "aui", "auid", "noaui", "senreuse", "nsenreuse", "boundscale", "noboundscale" }; -// auto end_iter = std::remove_if(tokens.begin(), tokens.end(), -// [&remove_tags](string &str)->bool{return (remove_tags.find(upper_cp(str)) != remove_tags.end() -// || remove_tags.find(lower_cp(str)) != remove_tags.end()); }); -// tokens.resize(std::distance(tokens.begin(), end_iter)); -// -// convert_ip(tokens[0], control_info.phiredswh); -// if (tokens.size() >= 2) { -// convert_ip(tokens[1], control_info.noptswitch); -// } -// if (tokens.size() >= 3) { -// convert_ip(tokens[2], control_info.splitswh); -// } -// -// } -// else if (sec_lnum == 7) -// { -// convert_ip(tokens[0], control_info.noptmax); -// convert_ip(tokens[1], control_info.phiredstp); -// convert_ip(tokens[2], control_info.nphistp); -// convert_ip(tokens[3], control_info.nphinored); -// convert_ip(tokens[4], control_info.relparstp); -// convert_ip(tokens[5], control_info.nrelpar); -// } -// else -// { -// other_lines[lnum] = line; -// } -// } -// else if (section == "SINGULAR VALUE DECOMPOSITION") -// { -// if (sec_lnum == 2) { -// convert_ip(tokens[0], svd_info.maxsing); -// convert_ip(tokens[1], svd_info.eigthresh); -// } -// else if (sec_lnum == 3) { -// convert_ip(tokens[0], svd_info.eigwrite); -// } -// else -// { -// other_lines[lnum] = line; -// } -// } -// else if (section == "PARAMETER GROUPS") -// { -// ParameterGroupRec pgi; -// name = tokens[0]; -// size_t n_tokens = tokens.size(); -// pgi.name = name; -// ctl_ordered_par_group_names.push_back(name); -// convert_ip(tokens[1], pgi.inctyp); -// convert_ip(tokens[2], pgi.derinc); -// convert_ip(tokens[3], pgi.derinclb); -// convert_ip(tokens[4], pgi.forcen); -// convert_ip(tokens[5], pgi.derincmul); -// convert_ip(tokens[6], pgi.dermthd); -// if (n_tokens >= 8) convert_ip(tokens[7], pgi.splitthresh); -// if (n_tokens >= 9) convert_ip(tokens[8], pgi.splitreldiff); -// base_group_info.insert_group(name, pgi); -// } -// else if (section == "PARAMETER DATA") -// { -// if (sec_lnum <= num_par) { -// double scale; -// double offset; -// ParameterRec pi; -// name = tokens[0]; -// trans_type = &tokens[1]; -// convert_ip(tokens[2], pi.chglim); -// convert_ip(tokens[3], pi.init_value); -// convert_ip(tokens[4], pi.lbnd); -// convert_ip(tokens[5], pi.ubnd); -// convert_ip(tokens[6], pi.group); -// convert_ip(tokens[7], scale); -// convert_ip(tokens[8], offset); -// if (control_info.numcom > 1) -// convert_ip(tokens[9], pi.dercom); -// else -// pi.dercom = 1; -// pi.scale = scale; -// pi.offset = offset; -// // add parameters to model parameter and paramter_info datasets -// ctl_ordered_par_names.push_back(name); -// if (*trans_type == "FIXED") -// { -// pi.tranform_type = ParameterRec::TRAN_TYPE::FIXED; -// } -// else if (*trans_type == "LOG") -// { -// pi.tranform_type = ParameterRec::TRAN_TYPE::LOG; -// n_adj_par++; -// } -// else if (*trans_type == "TIED") -// { -// pi.tranform_type = ParameterRec::TRAN_TYPE::TIED; -// } -// else if (*trans_type == "NONE") -// { -// pi.tranform_type = ParameterRec::TRAN_TYPE::NONE; -// n_adj_par++; -// } -// else -// { -// //pi.tranform_type = ParameterRec::TRAN_TYPE::NONE; -// //assert(true); -// //n_adj_par++; -// throw PestError("unrecognized partrans for par " + name + ": " + *trans_type); -// } -// ctl_parameter_info.insert(name, pi); -// ctl_parameters.insert(name, pi.init_value); -// base_group_info.insert_parameter_link(name, pi.group); -// -// // build appropriate transformations -// if (*trans_type == "FIXED") { -// t_fixed->insert(name, pi.init_value);} -// else if (*trans_type == "LOG") { -// t_log->insert(name); -// } -// if (offset!=0) { -// t_offset->insert(name, offset); -// } -// if (scale !=1) { -// t_scale->insert(name, scale); -// } -// } -// // Get rest of information for tied paramters -// else { -// name = tokens[0]; -// string name_tied = tokens[1]; -// double ratio = ctl_parameters[name] / ctl_parameters[name_tied]; -// t_tied->insert(name, pair(name_tied, ratio)); -// tied_names.insert(name_tied); -// } -// } -// else if (section == "OBSERVATION GROUPS") -// { -// string name = tokens[0]; -// if (tokens.size() > 1) -// { -// stringstream ss; -// ss << "observation covariance matrix detected for group '" << tokens[0] << "' - these are not supported...yet!"; -// string s = ss.str(); -// throw PestError(s); -// } -// ObservationGroupRec group_rec; -// observation_info.groups[name] = group_rec; -// vector::iterator is = find(ctl_ordered_obs_group_names.begin(), ctl_ordered_obs_group_names.end(), name); -// if (is == ctl_ordered_obs_group_names.end()) -// { -// ctl_ordered_obs_group_names.push_back(name); -// } -// } -// else if (section == "OBSERVATION DATA") -// { -// ObservationRec obs_i; -// name = tokens[0]; -// convert_ip(tokens[1], value); -// convert_ip(tokens[2], obs_i.weight); -// obs_i.group = tokens[3]; -// ctl_ordered_obs_names.push_back(name); -// observation_info.observations[name] = obs_i; -// observation_values.insert(name, value); -// } -// -// else if (section == "PRIOR INFORMATION") -// { -// //This section processes the prior information. It does not write out the -// //last prior infomration. THis is because it must check for line continuations -// if (!prior_info_string.empty() && tokens[0] != "&"){ -// pi_name_group = prior_info.AddRecord(prior_info_string); -// ctl_ordered_pi_names.push_back(pi_name_group.first); -// vector::iterator is = find(ctl_ordered_obs_group_names.begin(), ctl_ordered_obs_group_names.end(), pi_name_group.second); -// if (is == ctl_ordered_obs_group_names.end()) -// { -// ctl_ordered_obs_group_names.push_back(pi_name_group.second); -// } -// prior_info_string.clear(); -// } -// else if (tokens[0] == "&") { -// prior_info_string.append(" "); -// } -// prior_info_string.append(line_upper); -// } -// -// else if (section == "PRIOR INFORMATION" ) -// { -// //This section processes the prior information. It does not write out the -// //last prior infomration -// if (!prior_info_string.empty() && tokens[0] != "&") { -// pi_name_group = prior_info.AddRecord(prior_info_string); -// ctl_ordered_pi_names.push_back(pi_name_group.first); -// vector::iterator is = find(ctl_ordered_obs_group_names.begin(), ctl_ordered_obs_group_names.end(), pi_name_group.second); -// if (is == ctl_ordered_obs_group_names.end()) -// { -// ctl_ordered_obs_group_names.push_back(pi_name_group.second); -// } -// prior_info_string.clear(); -// } -// else if (tokens[0] != "&") { -// prior_info_string.append(" "); -// } -// prior_info_string.append(line); -// } -// else if (section == "MODEL COMMAND LINE" ) -// { -// model_exec_info.comline_vec.push_back(line); -// } -// else if (section == "MODEL INPUT/OUTPUT" ) -// { -// vector tokens_case_sen; -// tokenize(line, tokens_case_sen); -// if(i_tpl_ins < num_tpl_file) -// { -// model_exec_info.tplfile_vec.push_back(tokens_case_sen[0]); -// model_exec_info.inpfile_vec.push_back(tokens_case_sen[1]); -// } -// else -// { -// model_exec_info.insfile_vec.push_back(tokens_case_sen[0]); -// model_exec_info.outfile_vec.push_back(tokens_case_sen[1]); -// } -// ++i_tpl_ins; -// } -// else if (section == "REGULARISATION" || section=="REGULARIZATION" ) -// { -// if (sec_lnum == 1) { -// convert_ip(tokens[0], phimlim); -// convert_ip(tokens[1], phimaccept); -// fracphim = 0.0; -// if(tokens.size() >=3) convert_ip(tokens[2], fracphim); -// } -// else if (sec_lnum == 2) { -// convert_ip(tokens[0], wfinit); -// convert_ip(tokens[1], wfmin); -// convert_ip(tokens[2], wfmax); -// } -// else if (sec_lnum == 3) { -// int iregadj; -// convert_ip(tokens[0], wffac); -// convert_ip(tokens[1], wftol); -// if (tokens.size() > 2) -// { -// convert_ip(tokens[2], iregadj); -// if (iregadj == 1) reg_adj_grp_weights = true; -// } -// delete regul_scheme_ptr; -// regul_scheme_ptr = new DynamicRegularization(use_dynamic_reg, reg_adj_grp_weights, phimlim, -// phimaccept, fracphim, wfmin, wfmax, wffac, wftol, wfinit); -// } -// else -// { -// other_lines[lnum] = line; -// } -// } -// else if (section == "PARETO") -// { -// if (sec_lnum == 1) -// { -// convert_ip(tokens[0], pareto_info.obsgroup); -// } -// else if (sec_lnum == 2) -// { -// convert_ip(tokens[0], pareto_info.wf_start); -// convert_ip(tokens[1], pareto_info.wf_fin); -// convert_ip(tokens[2], pareto_info.wf_inc); -// } -// else if (sec_lnum == 3) -// { -// convert_ip(tokens[0], pareto_info.niter_start); -// convert_ip(tokens[1], pareto_info.niter_gen); -// convert_ip(tokens[2], pareto_info.niter_fin); -// } -// -// } -// else -// { -// other_lines[lnum] = line; -// } -// } -// -// // write out last prior information record -// if (!prior_info_string.empty()) -// { -// pi_name_group = prior_info.AddRecord(prior_info_string); -// ctl_ordered_pi_names.push_back(pi_name_group.first); -// vector::iterator is = find(ctl_ordered_obs_group_names.begin(), ctl_ordered_obs_group_names.end(), pi_name_group.second); -// if (is == ctl_ordered_obs_group_names.end()) -// { -// ctl_ordered_obs_group_names.push_back(pi_name_group.second); -// } -// prior_info_string.clear(); -// } -//#ifndef _DEBUG -// } -// catch (PestConversionError &e) { -// std::stringstream out; -// out << "Error parsing \"" << pst_filename << "\" on line number " << lnum << endl; -// out << e.what() << endl; -// e.add_front(out.str()); -// e.raise(); -// } -//#endif -// fin.close(); -// -// map arg_map, line_arg_map; -// for(vector::const_iterator b=pestpp_input.begin(),e=pestpp_input.end(); -// b!=e; ++b) { -// -// try -// { -// line_arg_map = pestpp_options.parse_plusplus_line(*b); -// } -// catch (...) -// { -// throw runtime_error("error parsing ++ line '" + line + "'"); -// } -// arg_map.insert(line_arg_map.begin(),line_arg_map.end()); -// } -// //check for not "accepted" args here... -// -// -// regul_scheme_ptr->set_max_reg_iter(pestpp_options.get_max_reg_iter()); -// -// if (pestpp_options.get_tie_by_group()) -// { -// cout << "Note: ++tie_by_group(true) - tying adjustable parameters by groups" << endl; -// f_rec << "Note: ++tie_by_group(true) - tying adjustable parameters by groups" << endl; -// map> group_map; -// string gname; -// ParameterRec::TRAN_TYPE tlog = ParameterRec::TRAN_TYPE::LOG, -// tnone = ParameterRec::TRAN_TYPE::NONE, -// ttied = ParameterRec::TRAN_TYPE::TIED; -// int new_n_adj_par = 0; -// for (auto pname : ctl_ordered_par_names) -// { -// if ((ctl_parameter_info.get_parameter_rec_ptr(pname)->tranform_type == tlog) || -// (ctl_parameter_info.get_parameter_rec_ptr(pname)->tranform_type == tnone)) -// { -// -// if (tied_names.find(pname) != tied_names.end()) -// { -// new_n_adj_par++; -// continue; -// } -// gname = ctl_parameter_info.get_parameter_rec_ptr(pname)->group; -// if (group_map.find(gname) == group_map.end()) -// group_map[gname] = vector(); -// group_map[gname].push_back(pname); -// } -// -// } -// string tie_to_name; -// vector to_tie_names; -// vector::const_iterator first, last; -// for (auto gm : group_map) -// { -// tie_to_name = gm.second[0]; -// new_n_adj_par++; -// first = gm.second.begin() + 1; -// last = gm.second.end(); -// to_tie_names = vector(first, last); -// for (auto pname : to_tie_names) -// { -// double ratio = ctl_parameters[pname] / ctl_parameters[tie_to_name]; -// t_tied->insert(pname, pair(tie_to_name, ratio)); -// ctl_parameter_info.get_parameter_rec_ptr_4_mod(pname)->tranform_type = ttied; -// } -// -// } -// f_rec << "-->number of adjustable parameters reduced from " << n_adj_par << " to " << new_n_adj_par << endl; -// cout << "-->number of adjustable parameters reduced from " << n_adj_par << " to " << new_n_adj_par << endl; -// n_adj_par = new_n_adj_par; -// if (tied_names.size() > 0) -// { -// f_rec << "-->existing adjustable parameters that others tie to have been maintained:" << endl; -// for (auto tname : tied_names) -// f_rec << tname << endl; -// } -// } -// -// return 0; -//} int Pest::process_ctl_file(ifstream& fin, string _pst_filename, ofstream& f_rec) { diff --git a/src/libs/pestpp_common/sequential_lp.cpp b/src/libs/pestpp_common/sequential_lp.cpp index a2f2e788..cf30b41f 100644 --- a/src/libs/pestpp_common/sequential_lp.cpp +++ b/src/libs/pestpp_common/sequential_lp.cpp @@ -82,7 +82,7 @@ void sequentialLP::initial_report() f_rec << "-->objective function sense (direction): " << optobjfunc.get_obj_sense() << endl; - f_rec << "-->number of decision variable: " << num_dec_vars() << endl; + f_rec << "-->number of decision variables: " << num_dec_vars() << endl; f_rec << "-->number of observation constraints: " << constraints.num_obs_constraints() << endl; f_rec << "-->number of prior information constraints: " << constraints.num_pi_constraints() << endl; if (iter_derinc_fac != 1.0) @@ -326,12 +326,41 @@ void sequentialLP::initialize_and_check() //if any decision vars have a transformation that is not allowed vector problem_trans; - for (auto &name : dv_names) - if (pest_scenario.get_ctl_parameter_info().get_parameter_rec_ptr(name)->tranform_type != ParameterRec::TRAN_TYPE::NONE) + vector temp_dv_names; + vector tied_dv_names; + //this should just skip over fixed dvs + for (auto &name : dv_names) + { + if (pest_scenario.get_ctl_parameter_info().get_parameter_rec_ptr(name)->tranform_type == ParameterRec::TRAN_TYPE::NONE) + temp_dv_names.push_back(name); + else if (pest_scenario.get_ctl_parameter_info().get_parameter_rec_ptr(name)->tranform_type == ParameterRec::TRAN_TYPE::TIED) + tied_dv_names.push_back(name); + else if (pest_scenario.get_ctl_parameter_info().get_parameter_rec_ptr(name)->tranform_type == ParameterRec::TRAN_TYPE::LOG) problem_trans.push_back(name); - if (problem_trans.size() > 0) - throw_sequentialLP_error("the following decision variables don't have 'none' type parameter transformation: ", problem_trans); + } + if (!problem_trans.empty()) + throw_sequentialLP_error("the following decision variables have 'log' type parameter transformation: ", problem_trans); + if (!tied_dv_names.empty()) + { + set sdv_names(temp_dv_names.begin(),temp_dv_names.end()); + auto send = sdv_names.end(); + problem_trans.clear(); + string tied_name; + map> tied_info = pest_scenario.get_base_par_tran_seq().get_tied_ptr()->get_items(); + for (auto& name : tied_dv_names) + { + if (sdv_names.find(tied_info[name].first) == send) + problem_trans.push_back(name); + } + if (!problem_trans.empty()) + { + throw_sequentialLP_error("the following 'tied' decision variables are not tied to a decision variable: ", problem_trans); + } + + } + dv_names = temp_dv_names; + temp_dv_names.clear(); current_constraints_sim = pest_scenario.get_ctl_observations(); constraints.initialize(dv_names, COIN_DBL_MAX); @@ -958,11 +987,11 @@ bool sequentialLP::make_upgrade_run(Parameters &upgrade_pars, Observations &upgr { cout << " --- running the model once with optimal decision variables --- " << endl; - int run_id = run_mgr_ptr->add_run(par_trans.ctl2model_cp(upgrade_pars)); + int run_id = run_mgr_ptr->add_run(par_trans.active_ctl2model_cp(upgrade_pars)); run_mgr_ptr->run(); bool success = run_mgr_ptr->get_run(run_id, upgrade_pars, upgrade_obs); if (success) - par_trans.model2ctl_ip(upgrade_pars); + par_trans.model2active_ctl_ip(upgrade_pars); return success; } @@ -1044,7 +1073,7 @@ void sequentialLP::iter_presolve() } else { - //make the intial base run + //make the initial base run cout << " --- running the model once with initial decision variables --- " << endl; int run_id = run_mgr_ptr->add_run(par_trans.ctl2model_cp(current_pars)); //this would be only for stack runs since the fosm runs should have been in the jco From 887e54b02a28ba2fd13e9011fef237fa1d906efc Mon Sep 17 00:00:00 2001 From: jwhite Date: Fri, 7 Jun 2024 11:00:15 -0600 Subject: [PATCH 27/27] added check that tied dvs arent tied to non dvs --- src/libs/pestpp_common/constraints.cpp | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/libs/pestpp_common/constraints.cpp b/src/libs/pestpp_common/constraints.cpp index f1f571c8..f90bc1f0 100644 --- a/src/libs/pestpp_common/constraints.cpp +++ b/src/libs/pestpp_common/constraints.cpp @@ -229,7 +229,7 @@ void Constraints::initialize(vector& ctl_ord_dec_var_names, double _dbl_ //current_constraints_sim_ptr = _current_constraints_sim_ptr; //same for dec var values //current_pars_and_dec_vars_ptr = _current_pars_and_dec_vars_ptr; - //check for a stack size arg - if postive, then use stacks for chances + //check for a stack size arg - if positive, then use stacks for chances stack_size = pest_scenario.get_pestpp_options().get_opt_stack_size(); //an existing parameter stack for chances string par_stack_name = pest_scenario.get_pestpp_options().get_opt_par_stack(); @@ -450,7 +450,12 @@ void Constraints::initialize(vector& ctl_ord_dec_var_names, double _dbl_ } set dec_set(ctl_ord_dec_var_names.begin(), ctl_ord_dec_var_names.end()); - for (auto& name : pest_scenario.get_ctl_ordered_par_names()) + vector prob_pars; + //disabled the exception for adj pars tied to dec vars + //string partied; + //map> tied_info = pest_scenario.get_base_par_tran_seq().get_tied_ptr()->get_items(); + + for (auto& name : pest_scenario.get_ctl_ordered_par_names()) { //if this parameter is not a decision var //if (find(start, end, name) == end) @@ -458,13 +463,26 @@ void Constraints::initialize(vector& ctl_ord_dec_var_names, double _dbl_ { ParameterRec::TRAN_TYPE tt = pest_scenario.get_ctl_parameter_info().get_parameter_rec_ptr(name)->tranform_type; if ((tt == ParameterRec::TRAN_TYPE::LOG) || (tt == ParameterRec::TRAN_TYPE::NONE)) - adj_par_names.push_back(name); + { + adj_par_names.push_back(name); + } +// else if (tt == ParameterRec::TRAN_TYPE::TIED) +// { +// partied = tied_info[name].first; +// if (dec_set.find(partied) != dec_set.end()) +// prob_pars.push_back(name); +// } } + } +// if (!prob_pars.empty()) +// { +// throw_constraints_error("the following adjustable pars are 'tied' to decision variables",prob_pars); +// } //------------------------------------------ - // --- chance constratints --- + // --- chance constraints --- //------------------------------------------ risk = pest_scenario.get_pestpp_options().get_opt_risk(); if (risk != 0.5)