From bed9892ef935a10d5fceb096be184b5cf00f5697 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Mon, 7 Aug 2023 16:41:41 +0100 Subject: [PATCH 01/46] Remove create_output_folder() function Fixes #186. --- src/HealthGPS.Console/configuration.cpp | 20 +------------------- src/HealthGPS.Console/configuration.h | 6 ------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index a987dad86..c04c79152 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -248,7 +248,7 @@ Configuration load_configuration(CommandOptions &options) { if (!fs::exists(config.output.folder)) { fmt::print(fg(fmt::color::dark_salmon), "\nCreating output folder: {} ...\n", config.output.folder); - if (!create_output_folder(config.output.folder)) { + if (!std::filesystem::create_directories(config.output.folder)) { throw std::runtime_error( fmt::format("Failed to create output folder: {}", config.output.folder)); } @@ -268,24 +268,6 @@ Configuration load_configuration(CommandOptions &options) { return config; } -bool create_output_folder(std::filesystem::path folder_path, unsigned int num_retries) { - using namespace std::chrono_literals; - for (unsigned int i = 1; i <= num_retries; i++) { - try { - if (std::filesystem::create_directories(folder_path)) { - return true; - } - } catch (const std::exception &ex) { - fmt::print(fg(fmt::color::red), "Failed to create output folder, attempt #{} - {}.\n", - i, ex.what()); - } - - std::this_thread::sleep_for(1000ms); - } - - return false; -} - std::vector get_diseases_info(core::Datastore &data_api, Configuration &config) { std::vector result; auto diseases = data_api.get_diseases(); diff --git a/src/HealthGPS.Console/configuration.h b/src/HealthGPS.Console/configuration.h index a44d6dd66..647c899c6 100644 --- a/src/HealthGPS.Console/configuration.h +++ b/src/HealthGPS.Console/configuration.h @@ -35,12 +35,6 @@ CommandOptions parse_arguments(cxxopts::Options &options, int &argc, char *argv[ /// @return The configuration file information Configuration load_configuration(CommandOptions &options); -/// @brief Creates the configuration output folder for result files -/// @param folder_path Full path to output folder -/// @param num_retries Number of attempts before giving up -/// @return true for successful creation, otherwise false -bool create_output_folder(std::filesystem::path folder_path, unsigned int num_retries = 3); - /// @brief Gets the collection of diseases that matches the selected input list /// @param data_api The back-end data store instance to be used. /// @param config User configuration file instance From 50505e113e310c2975cd26462194955747ed5b55 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Tue, 8 Aug 2023 09:53:31 +0100 Subject: [PATCH 02/46] Use std::filesystem::path cf. std::string in a few places --- src/HealthGPS.Console/configuration.cpp | 8 ++++---- src/HealthGPS.Console/model_parser.cpp | 8 ++++---- src/HealthGPS.Console/poco.h | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index c04c79152..89a091fd3 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -165,8 +165,8 @@ Configuration load_configuration(CommandOptions &options) { if (full_path.is_relative()) { full_path = options.config_file.parent_path() / config.file.name; if (fs::exists(full_path)) { - config.file.name = full_path.string(); - fmt::print("Input dataset file..: {}\n", config.file.name); + config.file.name = full_path; + fmt::print("Input dataset file..: {}\n", config.file.name.string()); } } @@ -188,7 +188,7 @@ Configuration load_configuration(CommandOptions &options) { full_path = options.config_file.parent_path() / model.second; if (fs::exists(full_path)) { model.second = full_path.string(); - fmt::print("Model: {:<7}, file: {}\n", model.first, model.second); + fmt::print("Model: {:<7}, file: {}\n", model.first, model.second.string()); } } @@ -202,7 +202,7 @@ Configuration load_configuration(CommandOptions &options) { for (auto &item : config.modelling.baseline_adjustment.file_names) { full_path = options.config_file.parent_path() / item.second; if (fs::exists(full_path)) { - item.second = full_path.string(); + item.second = full_path; fmt::print("{:<14}, file: {}\n", item.first, full_path.string()); } else { fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", diff --git a/src/HealthGPS.Console/model_parser.cpp b/src/HealthGPS.Console/model_parser.cpp index 4daa5d590..be59d0706 100644 --- a/src/HealthGPS.Console/model_parser.cpp +++ b/src/HealthGPS.Console/model_parser.cpp @@ -38,7 +38,7 @@ hgps::BaselineAdjustment load_baseline_adjustments(const poco::BaselineInfo &inf } } catch (const std::exception &ex) { fmt::print(fg(fmt::color::red), "Failed to parse adjustment file: {} or {}. {}\n", - male_filename, female_filename, ex.what()); + male_filename.string(), female_filename.string(), ex.what()); throw; } } @@ -275,10 +275,10 @@ load_risk_model_definition(const std::string &model_type, const poco::json &opt, throw std::invalid_argument(fmt::format("Unknown model type: {}", model_type)); } -poco::json load_json(const std::string &model_filename) { - std::ifstream ifs(model_filename, std::ifstream::in); +poco::json load_json(const std::filesystem::path &model_path) { + std::ifstream ifs(model_path, std::ifstream::in); if (!ifs.good()) { - throw std::invalid_argument(fmt::format("Model file: {} not found", model_filename)); + throw std::invalid_argument(fmt::format("Model file: {} not found", model_path.string())); } return poco::json::parse(ifs); diff --git a/src/HealthGPS.Console/poco.h b/src/HealthGPS.Console/poco.h index 75db4d288..e32d5f18c 100644 --- a/src/HealthGPS.Console/poco.h +++ b/src/HealthGPS.Console/poco.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -7,7 +8,7 @@ namespace host::poco { struct FileInfo { - std::string name; + std::filesystem::path name; std::string format; std::string delimiter; std::string encoding; @@ -29,7 +30,7 @@ struct BaselineInfo { std::string format; std::string delimiter; std::string encoding; - std::map file_names; + std::map file_names; }; struct RiskFactorInfo { @@ -40,7 +41,7 @@ struct RiskFactorInfo { struct ModellingInfo { std::vector risk_factors; - std::unordered_map risk_factor_models; + std::unordered_map risk_factor_models; BaselineInfo baseline_adjustment; }; From 27003246a8b9b185d94d122bda9387bd7382865a Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Tue, 8 Aug 2023 09:57:52 +0100 Subject: [PATCH 03/46] Drop support for v1 config files Fixes #198. --- src/HealthGPS.Console/configuration.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index 89a091fd3..d5cd17427 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -144,10 +144,10 @@ Configuration load_configuration(CommandOptions &options) { throw std::runtime_error("Invalid definition, file must have a schema version"); } - auto version = opt["version"].get(); - if (version != 1 && version != 2) { - throw std::runtime_error(fmt::format( - "configuration schema version: {} mismatch, supported: 1 and 2", version)); + const auto version = opt["version"].get(); + if (version != 2) { + throw std::runtime_error( + fmt::format("configuration schema version: {} mismatch, supported: 2", version)); } // application version @@ -155,12 +155,7 @@ Configuration load_configuration(CommandOptions &options) { config.app_version = PROJECT_VERSION; // input dataset file - auto dataset_key = "dataset"; - if (version == 1) { - dataset_key = "file"; - } - - config.file = opt["inputs"][dataset_key].get(); + config.file = opt["inputs"]["dataset"].get(); fs::path full_path = config.file.name; if (full_path.is_relative()) { full_path = options.config_file.parent_path() / config.file.name; From 358c76249e6b1ef422daffcfb64899afe626cb34 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Tue, 8 Aug 2023 11:28:46 +0100 Subject: [PATCH 04/46] load_configuration(): Remove some indentation --- src/HealthGPS.Console/configuration.cpp | 194 ++++++++++++------------ 1 file changed, 96 insertions(+), 98 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index d5cd17427..64c49e3bc 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -138,128 +138,126 @@ Configuration load_configuration(CommandOptions &options) { Configuration config; std::ifstream ifs(options.config_file, std::ifstream::in); - if (ifs) { - auto opt = json::parse(ifs); - if (!opt.contains("version")) { - throw std::runtime_error("Invalid definition, file must have a schema version"); - } + if (!ifs) { + throw std::runtime_error( + fmt::format("File {} doesn't exist.", options.config_file.string())); + } - const auto version = opt["version"].get(); - if (version != 2) { - throw std::runtime_error( - fmt::format("configuration schema version: {} mismatch, supported: 2", version)); + auto opt = json::parse(ifs); + if (!opt.contains("version")) { + throw std::runtime_error("Invalid definition, file must have a schema version"); + } + + const auto version = opt["version"].get(); + if (version != 2) { + throw std::runtime_error( + fmt::format("configuration schema version: {} mismatch, supported: 2", version)); + } + + // application version + config.app_name = PROJECT_NAME; + config.app_version = PROJECT_VERSION; + + // input dataset file + config.file = opt["inputs"]["dataset"].get(); + fs::path full_path = config.file.name; + if (full_path.is_relative()) { + full_path = options.config_file.parent_path() / config.file.name; + if (fs::exists(full_path)) { + config.file.name = full_path; + fmt::print("Input dataset file..: {}\n", config.file.name.string()); } + } - // application version - config.app_name = PROJECT_NAME; - config.app_version = PROJECT_VERSION; + if (!fs::exists(full_path)) { + fmt::print(fg(fmt::color::red), "\nInput data file: {} not found.\n", full_path.string()); + } + + // Settings and SES mapping + config.settings = opt["inputs"]["settings"].get(); + config.ses = opt["modelling"]["ses_model"].get(); - // input dataset file - config.file = opt["inputs"]["dataset"].get(); - fs::path full_path = config.file.name; + // Modelling information + config.modelling = opt["modelling"].get(); + + for (auto &model : config.modelling.risk_factor_models) { + full_path = model.second; if (full_path.is_relative()) { - full_path = options.config_file.parent_path() / config.file.name; + full_path = options.config_file.parent_path() / model.second; if (fs::exists(full_path)) { - config.file.name = full_path; - fmt::print("Input dataset file..: {}\n", config.file.name.string()); + model.second = full_path.string(); + fmt::print("Model: {:<7}, file: {}\n", model.first, model.second.string()); } } if (!fs::exists(full_path)) { - fmt::print(fg(fmt::color::red), "\nInput data file: {} not found.\n", + fmt::print(fg(fmt::color::red), "Model: {:<7}, file: {} not found.\n", model.first, full_path.string()); } + } - // Settings and SES mapping - config.settings = opt["inputs"]["settings"].get(); - config.ses = opt["modelling"]["ses_model"].get(); - - // Modelling information - config.modelling = opt["modelling"].get(); - - for (auto &model : config.modelling.risk_factor_models) { - full_path = model.second; - if (full_path.is_relative()) { - full_path = options.config_file.parent_path() / model.second; - if (fs::exists(full_path)) { - model.second = full_path.string(); - fmt::print("Model: {:<7}, file: {}\n", model.first, model.second.string()); - } - } - - if (!fs::exists(full_path)) { - fmt::print(fg(fmt::color::red), "Model: {:<7}, file: {} not found.\n", model.first, - full_path.string()); - } - } - - fmt::print("Baseline factor adjustment:\n"); - for (auto &item : config.modelling.baseline_adjustment.file_names) { - full_path = options.config_file.parent_path() / item.second; - if (fs::exists(full_path)) { - item.second = full_path; - fmt::print("{:<14}, file: {}\n", item.first, full_path.string()); - } else { - fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", - item.first, full_path.string()); - } - } - - // Run-time - opt["running"]["start_time"].get_to(config.start_time); - opt["running"]["stop_time"].get_to(config.stop_time); - opt["running"]["trial_runs"].get_to(config.trial_runs); - opt["running"]["sync_timeout_ms"].get_to(config.sync_timeout_ms); - auto seed = opt["running"]["seed"].get>(); - if (seed.size() > 0) { - config.custom_seed = seed[0]; + fmt::print("Baseline factor adjustment:\n"); + for (auto &item : config.modelling.baseline_adjustment.file_names) { + full_path = options.config_file.parent_path() / item.second; + if (fs::exists(full_path)) { + item.second = full_path; + fmt::print("{:<14}, file: {}\n", item.first, full_path.string()); + } else { + fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", + item.first, full_path.string()); } + } - opt["running"]["diseases"].get_to(config.diseases); - - // Intervention Policy - auto &interventions = opt["running"]["interventions"]; - if (!interventions["active_type_id"].is_null()) { - auto active_type = interventions["active_type_id"].get(); - auto &policy_types = interventions["types"]; - for (auto it = policy_types.begin(); it != policy_types.end(); ++it) { - if (core::case_insensitive::equals(it.key(), active_type)) { - config.intervention = it.value().get(); - config.intervention.identifier = core::to_lower(it.key()); - config.has_active_intervention = true; - break; - } - } + // Run-time + opt["running"]["start_time"].get_to(config.start_time); + opt["running"]["stop_time"].get_to(config.stop_time); + opt["running"]["trial_runs"].get_to(config.trial_runs); + opt["running"]["sync_timeout_ms"].get_to(config.sync_timeout_ms); + auto seed = opt["running"]["seed"].get>(); + if (seed.size() > 0) { + config.custom_seed = seed[0]; + } - if (!core::case_insensitive::equals(config.intervention.identifier, active_type)) { - throw std::runtime_error( - fmt::format("Unknown active intervention type identifier: {}", active_type)); + opt["running"]["diseases"].get_to(config.diseases); + + // Intervention Policy + auto &interventions = opt["running"]["interventions"]; + if (!interventions["active_type_id"].is_null()) { + auto active_type = interventions["active_type_id"].get(); + auto &policy_types = interventions["types"]; + for (auto it = policy_types.begin(); it != policy_types.end(); ++it) { + if (core::case_insensitive::equals(it.key(), active_type)) { + config.intervention = it.value().get(); + config.intervention.identifier = core::to_lower(it.key()); + config.has_active_intervention = true; + break; } } - config.job_id = options.job_id; - config.output = opt["output"].get(); - config.output.folder = expand_environment_variables(config.output.folder); - if (!fs::exists(config.output.folder)) { - fmt::print(fg(fmt::color::dark_salmon), "\nCreating output folder: {} ...\n", - config.output.folder); - if (!std::filesystem::create_directories(config.output.folder)) { - throw std::runtime_error( - fmt::format("Failed to create output folder: {}", config.output.folder)); - } + if (!core::case_insensitive::equals(config.intervention.identifier, active_type)) { + throw std::runtime_error( + fmt::format("Unknown active intervention type identifier: {}", active_type)); } + } - // verbosity - config.verbosity = core::VerboseMode::none; - if (options.verbose) { - config.verbosity = core::VerboseMode::verbose; + config.job_id = options.job_id; + config.output = opt["output"].get(); + config.output.folder = expand_environment_variables(config.output.folder); + if (!fs::exists(config.output.folder)) { + fmt::print(fg(fmt::color::dark_salmon), "\nCreating output folder: {} ...\n", + config.output.folder); + if (!std::filesystem::create_directories(config.output.folder)) { + throw std::runtime_error( + fmt::format("Failed to create output folder: {}", config.output.folder)); } - } else { - std::cout << fmt::format("File {} doesn't exist.", options.config_file.string()) - << std::endl; } - ifs.close(); + // verbosity + config.verbosity = core::VerboseMode::none; + if (options.verbose) { + config.verbosity = core::VerboseMode::verbose; + } + return config; } From a2fd569ebda5efde25b2454b5e044e4803a80354 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Tue, 8 Aug 2023 12:31:34 +0100 Subject: [PATCH 05/46] Make separate function for rebasing paths --- src/HealthGPS.Console/configuration.cpp | 56 +++------ src/HealthGPS.Console/jsonparser.cpp | 144 +++++++++++++++--------- src/HealthGPS.Console/jsonparser.h | 45 ++++++-- 3 files changed, 144 insertions(+), 101 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index 64c49e3bc..4c0b94f97 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -158,19 +158,17 @@ Configuration load_configuration(CommandOptions &options) { config.app_name = PROJECT_NAME; config.app_version = PROJECT_VERSION; - // input dataset file - config.file = opt["inputs"]["dataset"].get(); - fs::path full_path = config.file.name; - if (full_path.is_relative()) { - full_path = options.config_file.parent_path() / config.file.name; - if (fs::exists(full_path)) { - config.file.name = full_path; - fmt::print("Input dataset file..: {}\n", config.file.name.string()); - } - } + // Base dir for relative paths + const auto config_dir = options.config_file.parent_path(); + bool success = true; - if (!fs::exists(full_path)) { - fmt::print(fg(fmt::color::red), "\nInput data file: {} not found.\n", full_path.string()); + // input dataset file + try { + config.file = get_file_info(opt["inputs"]["dataset"], config_dir); + fmt::print("Input dataset file: {}\n", config.file.name.string()); + } catch (const std::exception &e) { + success = false; + fmt::print(fg(fmt::color::red), "Could not load dataset file: {}\n", e.what()); } // Settings and SES mapping @@ -178,34 +176,16 @@ Configuration load_configuration(CommandOptions &options) { config.ses = opt["modelling"]["ses_model"].get(); // Modelling information - config.modelling = opt["modelling"].get(); - - for (auto &model : config.modelling.risk_factor_models) { - full_path = model.second; - if (full_path.is_relative()) { - full_path = options.config_file.parent_path() / model.second; - if (fs::exists(full_path)) { - model.second = full_path.string(); - fmt::print("Model: {:<7}, file: {}\n", model.first, model.second.string()); - } - } - - if (!fs::exists(full_path)) { - fmt::print(fg(fmt::color::red), "Model: {:<7}, file: {} not found.\n", model.first, - full_path.string()); - } + try { + config.modelling = get_modelling_info(opt["modelling"], config_dir); + } catch (const std::exception &e) { + success = false; + fmt::print(fg(fmt::color::red), "Could not load modelling info: {}\n", e.what()); } - fmt::print("Baseline factor adjustment:\n"); - for (auto &item : config.modelling.baseline_adjustment.file_names) { - full_path = options.config_file.parent_path() / item.second; - if (fs::exists(full_path)) { - item.second = full_path; - fmt::print("{:<14}, file: {}\n", item.first, full_path.string()); - } else { - fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", - item.first, full_path.string()); - } + if (!success) { + // TODO: Check more things before aborting + throw std::runtime_error("Could not load config"); } // Run-time diff --git a/src/HealthGPS.Console/jsonparser.cpp b/src/HealthGPS.Console/jsonparser.cpp index 771e6d7a9..d9b033aef 100644 --- a/src/HealthGPS.Console/jsonparser.cpp +++ b/src/HealthGPS.Console/jsonparser.cpp @@ -1,6 +1,92 @@ #include "jsonparser.h" +#include +#include namespace host::poco { +void rebase_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { + if (path.is_relative()) { + path = std::filesystem::weakly_canonical(base_dir / path); + } + + if (!std::filesystem::exists(path)) { + throw std::invalid_argument{fmt::format("Path does not exist: {}", path.string())}; + } +} + +std::filesystem::path get_valid_path(const json &j, const std::filesystem::path &base_dir) { + auto path = j.get(); + rebase_path(path, base_dir); + return path; +} + +FileInfo get_file_info(const json &j, const std::filesystem::path &base_dir) { + FileInfo info; + info.name = get_valid_path(j["name"], base_dir); + j.at("format").get_to(info.format); + j.at("delimiter").get_to(info.delimiter); + j.at("columns").get_to(info.columns); + return info; +} + +BaselineInfo get_baseline_info(const json &j, const std::filesystem::path &base_dir) { + BaselineInfo info; + j.at("format").get_to(info.format); + j.at("delimiter").get_to(info.delimiter); + j.at("encoding").get_to(info.encoding); + j.at("file_names").get_to(info.file_names); + + // Rebase paths and check for errors + bool success = true; + for (auto &[name, path] : info.file_names) { + try { + rebase_path(path, base_dir); + fmt::print("{:<14}, file: {}\n", name, path.string()); + } catch (const std::invalid_argument &) { + fmt::print(fg(fmt::color::red), "Could not find file: {}\n", path.string()); + success = false; + } + } + + if (!success) { + throw std::invalid_argument{"One or more files could not be found"}; + } + + return info; +} + +ModellingInfo get_modelling_info(const json &j, const std::filesystem::path &base_dir) { + bool success = true; + + ModellingInfo info; + j.at("risk_factors").get_to(info.risk_factors); + j.at("risk_factor_models").get_to(info.risk_factor_models); + + // Rebase paths and check for errors + for (auto &[type, path] : info.risk_factor_models) { + try { + rebase_path(path, base_dir); + fmt::print("{:<14}, file: {}\n", type, path.string()); + } catch (const std::invalid_argument &) { + success = false; + fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", type, + path.string()); + } + } + + try { + info.baseline_adjustment = get_baseline_info(j["baseline_adjustments"], base_dir); + } catch (const std::exception &e) { + success = false; + fmt::print(fmt::fg(fmt::color::red), "Could not load baseline adjustment: {}\n", e.what()); + } + + if (!success) { + throw std::invalid_argument("Could not load modelling info"); + } + + return info; +} + //-------------------------------------------------------- // Risk Model JSON serialisation / de-serialisation //-------------------------------------------------------- @@ -62,19 +148,6 @@ void from_json(const json &j, HierarchicalLevelInfo &p) { // Options JSON serialisation / de-serialisation //-------------------------------------------------------- -// Data file information -void to_json(json &j, const FileInfo &p) { - j = json{ - {"name", p.name}, {"format", p.format}, {"delimiter", p.delimiter}, {"columns", p.columns}}; -} - -void from_json(const json &j, FileInfo &p) { - j.at("name").get_to(p.name); - j.at("format").get_to(p.format); - j.at("delimiter").get_to(p.delimiter); - j.at("columns").get_to(p.columns); -} - // Settings Information void to_json(json &j, const SettingsInfo &p) { j = json{{"country_code", p.country}, @@ -88,52 +161,21 @@ void from_json(const json &j, SettingsInfo &p) { j.at("age_range").get_to(p.age_range); } -// SES Model Information -void to_json(json &j, const SESInfo &p) { - j = json{{"function_name", p.function}, {"function_parameters", p.parameters}}; -} - -void from_json(const json &j, SESInfo &p) { - j.at("function_name").get_to(p.function); - j.at("function_parameters").get_to(p.parameters); -} - -// Baseline scenario adjustments -void to_json(json &j, const BaselineInfo &p) { - j = json{{"format", p.format}, - {"delimiter", p.delimiter}, - {"encoding", p.encoding}, - {"file_names", p.file_names}}; -} - -void from_json(const json &j, BaselineInfo &p) { - j.at("format").get_to(p.format); - j.at("delimiter").get_to(p.delimiter); - j.at("encoding").get_to(p.encoding); - j.at("file_names").get_to(p.file_names); -} - -// Risk Factor Modelling -void to_json(json &j, const RiskFactorInfo &p) { - j = json{{"name", p.name}, {"level", p.level}, {"range", p.range}}; -} - +// Risk factor modelling void from_json(const json &j, RiskFactorInfo &p) { j.at("name").get_to(p.name); j.at("level").get_to(p.level); j.at("range").get_to(p.range); } -void to_json(json &j, const ModellingInfo &p) { - j = json{{"risk_factors", p.risk_factors}, - {"risk_factor_models", p.risk_factor_models}, - {"baseline_adjustments", p.baseline_adjustment}}; +// SES Model Information +void to_json(json &j, const SESInfo &p) { + j = json{{"function_name", p.function}, {"function_parameters", p.parameters}}; } -void from_json(const json &j, ModellingInfo &p) { - j.at("risk_factors").get_to(p.risk_factors); - j.at("risk_factor_models").get_to(p.risk_factor_models); - j.at("baseline_adjustments").get_to(p.baseline_adjustment); +void from_json(const json &j, SESInfo &p) { + j.at("function_name").get_to(p.function); + j.at("function_parameters").get_to(p.parameters); } void to_json(json &j, const VariableInfo &p) { diff --git a/src/HealthGPS.Console/jsonparser.h b/src/HealthGPS.Console/jsonparser.h index ece133fc6..51ea4b79e 100644 --- a/src/HealthGPS.Console/jsonparser.h +++ b/src/HealthGPS.Console/jsonparser.h @@ -2,6 +2,7 @@ #include "options.h" #include "riskmodel.h" +#include #include #include @@ -15,6 +16,38 @@ namespace host::poco { /// for details about the contents and code structure in this file. using json = nlohmann::json; +/// @brief Get a path, based on base_dir, and check if it exists +/// @param j Input JSON +/// @param base_dir Base folder +/// @return An absolute path, assuming that base_dir is the base if relative +/// @throw json::type_error: Invalid JSON types +/// @throw std::invalid_argument: Path does not exist +std::filesystem::path get_valid_path(const json &j, const std::filesystem::path &base_dir); + +/// @brief Load FileInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return FileInfo +/// @throw json::type_error: Invalid JSON types +/// @throw std::invalid_argument: Path does not exist +FileInfo get_file_info(const json &j, const std::filesystem::path &base_dir); + +/// @brief Load BaselineInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return BaselineInfo +/// @throw json::type_error: Invalid JSON types +/// @throw std::invalid_argument: Path does not exist +BaselineInfo get_baseline_info(const json &j, const std::filesystem::path &base_dir); + +/// @brief Load ModellingInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return ModellingInfo +/// @throw json::type_error: Invalid JSON types +/// @throw std::invalid_argument: Path does not exist +ModellingInfo get_modelling_info(const json &j, const std::filesystem::path &base_dir); + //-------------------------------------------------------- // Full risk factor model POCO types mapping //-------------------------------------------------------- @@ -37,10 +70,6 @@ void from_json(const json &j, HierarchicalLevelInfo &p); // Configuration sections POCO types mapping //-------------------------------------------------------- -// Data file information -void to_json(json &j, const FileInfo &p); -void from_json(const json &j, FileInfo &p); - // Settings Information void to_json(json &j, const SettingsInfo &p); void from_json(const json &j, SettingsInfo &p); @@ -49,20 +78,12 @@ void from_json(const json &j, SettingsInfo &p); void to_json(json &j, const SESInfo &p); void from_json(const json &j, SESInfo &p); -// Baseline scenario adjustments -void to_json(json &j, const BaselineInfo &p); -void from_json(const json &j, BaselineInfo &p); - // Lite risk factors models (Energy Balance Model) -void to_json(json &j, const RiskFactorInfo &p); void from_json(const json &j, RiskFactorInfo &p); void to_json(json &j, const VariableInfo &p); void from_json(const json &j, VariableInfo &p); -void to_json(json &j, const ModellingInfo &p); -void from_json(const json &j, ModellingInfo &p); - void to_json(json &j, const FactorDynamicEquationInfo &p); void from_json(const json &j, FactorDynamicEquationInfo &p); From c66c0b81602c44bd6e22ec8079dbfe2afd5436ca Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Tue, 8 Aug 2023 13:38:31 +0100 Subject: [PATCH 06/46] Put new functions for loading JSON into configuration.cpp --- src/HealthGPS.Console/configuration.cpp | 109 ++++++++++++++++++++++++ src/HealthGPS.Console/jsonparser.cpp | 86 ------------------- src/HealthGPS.Console/jsonparser.h | 33 ------- 3 files changed, 109 insertions(+), 119 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index 4c0b94f97..4f07bbfc1 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -11,6 +11,7 @@ #include "HealthGPS/physical_activity_scenario.h" #include "HealthGPS/simple_policy_scenario.h" +#include "HealthGPS.Core/poco.h" #include "HealthGPS.Core/scoped_timer.h" #include @@ -30,6 +31,114 @@ namespace host { using namespace hgps; +using json = nlohmann::json; +void rebase_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { + if (path.is_relative()) { + path = std::filesystem::weakly_canonical(base_dir / path); + } + + if (!std::filesystem::exists(path)) { + throw std::invalid_argument{fmt::format("Path does not exist: {}", path.string())}; + } +} + +/// @brief Get a path, based on base_dir, and check if it exists +/// @param j Input JSON +/// @param base_dir Base folder +/// @return An absolute path, assuming that base_dir is the base if relative +/// @throw json::type_error: Invalid JSON types +/// @throw std::invalid_argument: Path does not exist +std::filesystem::path get_valid_path(const json &j, const std::filesystem::path &base_dir) { + auto path = j.get(); + rebase_path(path, base_dir); + return path; +} + +/// @brief Load FileInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return FileInfo +/// @throw json::type_error: Invalid JSON types +/// @throw std::invalid_argument: Path does not exist +auto get_file_info(const json &j, const std::filesystem::path &base_dir) { + poco::FileInfo info; + info.name = get_valid_path(j["name"], base_dir); + j.at("format").get_to(info.format); + j.at("delimiter").get_to(info.delimiter); + j.at("columns").get_to(info.columns); + return info; +} + +/// @brief Load BaselineInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return BaselineInfo +/// @throw json::type_error: Invalid JSON types +/// @throw std::invalid_argument: Path does not exist +auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { + poco::BaselineInfo info; + j.at("format").get_to(info.format); + j.at("delimiter").get_to(info.delimiter); + j.at("encoding").get_to(info.encoding); + j.at("file_names").get_to(info.file_names); + + // Rebase paths and check for errors + bool success = true; + for (auto &[name, path] : info.file_names) { + try { + rebase_path(path, base_dir); + fmt::print("{:<14}, file: {}\n", name, path.string()); + } catch (const std::invalid_argument &) { + fmt::print(fg(fmt::color::red), "Could not find file: {}\n", path.string()); + success = false; + } + } + + if (!success) { + throw std::invalid_argument{"One or more files could not be found"}; + } + + return info; +} + +/// @brief Load ModellingInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return ModellingInfo +/// @throw json::type_error: Invalid JSON types +/// @throw std::invalid_argument: Path does not exist +auto get_modelling_info(const json &j, const std::filesystem::path &base_dir) { + bool success = true; + + poco::ModellingInfo info; + j.at("risk_factors").get_to(info.risk_factors); + j.at("risk_factor_models").get_to(info.risk_factor_models); + + // Rebase paths and check for errors + for (auto &[type, path] : info.risk_factor_models) { + try { + rebase_path(path, base_dir); + fmt::print("{:<14}, file: {}\n", type, path.string()); + } catch (const std::invalid_argument &) { + success = false; + fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", type, + path.string()); + } + } + + try { + info.baseline_adjustment = get_baseline_info(j["baseline_adjustments"], base_dir); + } catch (const std::exception &e) { + success = false; + fmt::print(fmt::fg(fmt::color::red), "Could not load baseline adjustment: {}\n", e.what()); + } + + if (!success) { + throw std::invalid_argument("Could not load modelling info"); + } + + return info; +} std::string get_time_now_str() { auto tp = std::chrono::system_clock::now(); diff --git a/src/HealthGPS.Console/jsonparser.cpp b/src/HealthGPS.Console/jsonparser.cpp index d9b033aef..074403d55 100644 --- a/src/HealthGPS.Console/jsonparser.cpp +++ b/src/HealthGPS.Console/jsonparser.cpp @@ -1,92 +1,6 @@ #include "jsonparser.h" -#include -#include namespace host::poco { -void rebase_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { - if (path.is_relative()) { - path = std::filesystem::weakly_canonical(base_dir / path); - } - - if (!std::filesystem::exists(path)) { - throw std::invalid_argument{fmt::format("Path does not exist: {}", path.string())}; - } -} - -std::filesystem::path get_valid_path(const json &j, const std::filesystem::path &base_dir) { - auto path = j.get(); - rebase_path(path, base_dir); - return path; -} - -FileInfo get_file_info(const json &j, const std::filesystem::path &base_dir) { - FileInfo info; - info.name = get_valid_path(j["name"], base_dir); - j.at("format").get_to(info.format); - j.at("delimiter").get_to(info.delimiter); - j.at("columns").get_to(info.columns); - return info; -} - -BaselineInfo get_baseline_info(const json &j, const std::filesystem::path &base_dir) { - BaselineInfo info; - j.at("format").get_to(info.format); - j.at("delimiter").get_to(info.delimiter); - j.at("encoding").get_to(info.encoding); - j.at("file_names").get_to(info.file_names); - - // Rebase paths and check for errors - bool success = true; - for (auto &[name, path] : info.file_names) { - try { - rebase_path(path, base_dir); - fmt::print("{:<14}, file: {}\n", name, path.string()); - } catch (const std::invalid_argument &) { - fmt::print(fg(fmt::color::red), "Could not find file: {}\n", path.string()); - success = false; - } - } - - if (!success) { - throw std::invalid_argument{"One or more files could not be found"}; - } - - return info; -} - -ModellingInfo get_modelling_info(const json &j, const std::filesystem::path &base_dir) { - bool success = true; - - ModellingInfo info; - j.at("risk_factors").get_to(info.risk_factors); - j.at("risk_factor_models").get_to(info.risk_factor_models); - - // Rebase paths and check for errors - for (auto &[type, path] : info.risk_factor_models) { - try { - rebase_path(path, base_dir); - fmt::print("{:<14}, file: {}\n", type, path.string()); - } catch (const std::invalid_argument &) { - success = false; - fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", type, - path.string()); - } - } - - try { - info.baseline_adjustment = get_baseline_info(j["baseline_adjustments"], base_dir); - } catch (const std::exception &e) { - success = false; - fmt::print(fmt::fg(fmt::color::red), "Could not load baseline adjustment: {}\n", e.what()); - } - - if (!success) { - throw std::invalid_argument("Could not load modelling info"); - } - - return info; -} - //-------------------------------------------------------- // Risk Model JSON serialisation / de-serialisation //-------------------------------------------------------- diff --git a/src/HealthGPS.Console/jsonparser.h b/src/HealthGPS.Console/jsonparser.h index 51ea4b79e..041ee217f 100644 --- a/src/HealthGPS.Console/jsonparser.h +++ b/src/HealthGPS.Console/jsonparser.h @@ -2,7 +2,6 @@ #include "options.h" #include "riskmodel.h" -#include #include #include @@ -16,38 +15,6 @@ namespace host::poco { /// for details about the contents and code structure in this file. using json = nlohmann::json; -/// @brief Get a path, based on base_dir, and check if it exists -/// @param j Input JSON -/// @param base_dir Base folder -/// @return An absolute path, assuming that base_dir is the base if relative -/// @throw json::type_error: Invalid JSON types -/// @throw std::invalid_argument: Path does not exist -std::filesystem::path get_valid_path(const json &j, const std::filesystem::path &base_dir); - -/// @brief Load FileInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return FileInfo -/// @throw json::type_error: Invalid JSON types -/// @throw std::invalid_argument: Path does not exist -FileInfo get_file_info(const json &j, const std::filesystem::path &base_dir); - -/// @brief Load BaselineInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return BaselineInfo -/// @throw json::type_error: Invalid JSON types -/// @throw std::invalid_argument: Path does not exist -BaselineInfo get_baseline_info(const json &j, const std::filesystem::path &base_dir); - -/// @brief Load ModellingInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return ModellingInfo -/// @throw json::type_error: Invalid JSON types -/// @throw std::invalid_argument: Path does not exist -ModellingInfo get_modelling_info(const json &j, const std::filesystem::path &base_dir); - //-------------------------------------------------------- // Full risk factor model POCO types mapping //-------------------------------------------------------- From d9a8ab89ca09cbb84012bba6d4aac7a2b1555b66 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Tue, 8 Aug 2023 13:52:33 +0100 Subject: [PATCH 07/46] Add ConfigurationError class and use in a few places --- src/HealthGPS.Console/configuration.cpp | 29 ++++++++++++++----------- src/HealthGPS.Console/configuration.h | 9 ++++++++ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index 4f07bbfc1..e0a1383c5 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -32,13 +32,16 @@ namespace host { using namespace hgps; using json = nlohmann::json; + +ConfigurationError::ConfigurationError(const std::string &msg) : std::runtime_error{msg} {} + void rebase_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { if (path.is_relative()) { path = std::filesystem::weakly_canonical(base_dir / path); } if (!std::filesystem::exists(path)) { - throw std::invalid_argument{fmt::format("Path does not exist: {}", path.string())}; + throw ConfigurationError{fmt::format("Path does not exist: {}", path.string())}; } } @@ -47,7 +50,7 @@ void rebase_path(std::filesystem::path &path, const std::filesystem::path &base_ /// @param base_dir Base folder /// @return An absolute path, assuming that base_dir is the base if relative /// @throw json::type_error: Invalid JSON types -/// @throw std::invalid_argument: Path does not exist +/// @throw ConfigurationError: Path does not exist std::filesystem::path get_valid_path(const json &j, const std::filesystem::path &base_dir) { auto path = j.get(); rebase_path(path, base_dir); @@ -59,7 +62,7 @@ std::filesystem::path get_valid_path(const json &j, const std::filesystem::path /// @param base_dir Base folder /// @return FileInfo /// @throw json::type_error: Invalid JSON types -/// @throw std::invalid_argument: Path does not exist +/// @throw ConfigurationError: Path does not exist auto get_file_info(const json &j, const std::filesystem::path &base_dir) { poco::FileInfo info; info.name = get_valid_path(j["name"], base_dir); @@ -74,7 +77,7 @@ auto get_file_info(const json &j, const std::filesystem::path &base_dir) { /// @param base_dir Base folder /// @return BaselineInfo /// @throw json::type_error: Invalid JSON types -/// @throw std::invalid_argument: Path does not exist +/// @throw ConfigurationError: One or more files could not be found auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { poco::BaselineInfo info; j.at("format").get_to(info.format); @@ -88,14 +91,14 @@ auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { try { rebase_path(path, base_dir); fmt::print("{:<14}, file: {}\n", name, path.string()); - } catch (const std::invalid_argument &) { + } catch (const ConfigurationError &) { fmt::print(fg(fmt::color::red), "Could not find file: {}\n", path.string()); success = false; } } if (!success) { - throw std::invalid_argument{"One or more files could not be found"}; + throw ConfigurationError{"One or more files could not be found"}; } return info; @@ -106,7 +109,7 @@ auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { /// @param base_dir Base folder /// @return ModellingInfo /// @throw json::type_error: Invalid JSON types -/// @throw std::invalid_argument: Path does not exist +/// @throw ConfigurationError: Could not load modelling info auto get_modelling_info(const json &j, const std::filesystem::path &base_dir) { bool success = true; @@ -119,7 +122,7 @@ auto get_modelling_info(const json &j, const std::filesystem::path &base_dir) { try { rebase_path(path, base_dir); fmt::print("{:<14}, file: {}\n", type, path.string()); - } catch (const std::invalid_argument &) { + } catch (const ConfigurationError &) { success = false; fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", type, path.string()); @@ -134,7 +137,7 @@ auto get_modelling_info(const json &j, const std::filesystem::path &base_dir) { } if (!success) { - throw std::invalid_argument("Could not load modelling info"); + throw ConfigurationError("Could not load modelling info"); } return info; @@ -248,19 +251,19 @@ Configuration load_configuration(CommandOptions &options) { Configuration config; std::ifstream ifs(options.config_file, std::ifstream::in); if (!ifs) { - throw std::runtime_error( + throw ConfigurationError( fmt::format("File {} doesn't exist.", options.config_file.string())); } auto opt = json::parse(ifs); if (!opt.contains("version")) { - throw std::runtime_error("Invalid definition, file must have a schema version"); + throw ConfigurationError("Invalid definition, file must have a schema version"); } const auto version = opt["version"].get(); if (version != 2) { - throw std::runtime_error( - fmt::format("configuration schema version: {} mismatch, supported: 2", version)); + throw ConfigurationError( + fmt::format("Configuration schema version: {} mismatch, supported: 2", version)); } // application version diff --git a/src/HealthGPS.Console/configuration.h b/src/HealthGPS.Console/configuration.h index 647c899c6..d2d7854d3 100644 --- a/src/HealthGPS.Console/configuration.h +++ b/src/HealthGPS.Console/configuration.h @@ -11,7 +11,16 @@ #include "options.h" #include "result_file_writer.h" +#include + namespace host { + +/// @brief Represents an error that occurred with the format of a config file +class ConfigurationError : public std::runtime_error { + public: + ConfigurationError(const std::string &msg); +}; + /// @brief Get a string representation of current system time /// @return The system time as string std::string get_time_now_str(); From 100ffd77e6d1da7202eb64e7ed3c90ee6afab20f Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Tue, 8 Aug 2023 15:03:28 +0100 Subject: [PATCH 08/46] Remove unused field --- src/HealthGPS.Console/poco.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/HealthGPS.Console/poco.h b/src/HealthGPS.Console/poco.h index e32d5f18c..a7a29b1d2 100644 --- a/src/HealthGPS.Console/poco.h +++ b/src/HealthGPS.Console/poco.h @@ -11,7 +11,6 @@ struct FileInfo { std::filesystem::path name; std::string format; std::string delimiter; - std::string encoding; std::map columns; }; From d1cc5712ef5b3149bba738885b238e04c10e69dc Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Tue, 8 Aug 2023 17:09:14 +0100 Subject: [PATCH 09/46] Clean up and add extra error checking to config file loading --- src/HealthGPS.Console/configuration.cpp | 526 ++++++++++++++++-------- src/HealthGPS.Console/configuration.h | 2 +- src/HealthGPS.Console/program.cpp | 13 +- 3 files changed, 364 insertions(+), 177 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index e0a1383c5..e76d53ebd 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -33,116 +33,6 @@ namespace host { using namespace hgps; using json = nlohmann::json; -ConfigurationError::ConfigurationError(const std::string &msg) : std::runtime_error{msg} {} - -void rebase_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { - if (path.is_relative()) { - path = std::filesystem::weakly_canonical(base_dir / path); - } - - if (!std::filesystem::exists(path)) { - throw ConfigurationError{fmt::format("Path does not exist: {}", path.string())}; - } -} - -/// @brief Get a path, based on base_dir, and check if it exists -/// @param j Input JSON -/// @param base_dir Base folder -/// @return An absolute path, assuming that base_dir is the base if relative -/// @throw json::type_error: Invalid JSON types -/// @throw ConfigurationError: Path does not exist -std::filesystem::path get_valid_path(const json &j, const std::filesystem::path &base_dir) { - auto path = j.get(); - rebase_path(path, base_dir); - return path; -} - -/// @brief Load FileInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return FileInfo -/// @throw json::type_error: Invalid JSON types -/// @throw ConfigurationError: Path does not exist -auto get_file_info(const json &j, const std::filesystem::path &base_dir) { - poco::FileInfo info; - info.name = get_valid_path(j["name"], base_dir); - j.at("format").get_to(info.format); - j.at("delimiter").get_to(info.delimiter); - j.at("columns").get_to(info.columns); - return info; -} - -/// @brief Load BaselineInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return BaselineInfo -/// @throw json::type_error: Invalid JSON types -/// @throw ConfigurationError: One or more files could not be found -auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { - poco::BaselineInfo info; - j.at("format").get_to(info.format); - j.at("delimiter").get_to(info.delimiter); - j.at("encoding").get_to(info.encoding); - j.at("file_names").get_to(info.file_names); - - // Rebase paths and check for errors - bool success = true; - for (auto &[name, path] : info.file_names) { - try { - rebase_path(path, base_dir); - fmt::print("{:<14}, file: {}\n", name, path.string()); - } catch (const ConfigurationError &) { - fmt::print(fg(fmt::color::red), "Could not find file: {}\n", path.string()); - success = false; - } - } - - if (!success) { - throw ConfigurationError{"One or more files could not be found"}; - } - - return info; -} - -/// @brief Load ModellingInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return ModellingInfo -/// @throw json::type_error: Invalid JSON types -/// @throw ConfigurationError: Could not load modelling info -auto get_modelling_info(const json &j, const std::filesystem::path &base_dir) { - bool success = true; - - poco::ModellingInfo info; - j.at("risk_factors").get_to(info.risk_factors); - j.at("risk_factor_models").get_to(info.risk_factor_models); - - // Rebase paths and check for errors - for (auto &[type, path] : info.risk_factor_models) { - try { - rebase_path(path, base_dir); - fmt::print("{:<14}, file: {}\n", type, path.string()); - } catch (const ConfigurationError &) { - success = false; - fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", type, - path.string()); - } - } - - try { - info.baseline_adjustment = get_baseline_info(j["baseline_adjustments"], base_dir); - } catch (const std::exception &e) { - success = false; - fmt::print(fmt::fg(fmt::color::red), "Could not load baseline adjustment: {}\n", e.what()); - } - - if (!success) { - throw ConfigurationError("Could not load modelling info"); - } - - return info; -} - std::string get_time_now_str() { auto tp = std::chrono::system_clock::now(); return fmt::format("{0:%F %H:%M:}{1:%S} {0:%Z}", tp, tp.time_since_epoch()); @@ -243,27 +133,357 @@ CommandOptions parse_arguments(cxxopts::Options &options, int &argc, char *argv[ return cmd; } -Configuration load_configuration(CommandOptions &options) { +ConfigurationError::ConfigurationError(const std::string &msg) : std::runtime_error{msg} {} + +auto get_key(const json &j, const std::string &key) { + try { + return j.at(key); + } catch (const std::out_of_range &) { + fmt::print(fg(fmt::color::red), "Missing key \"{}\"\n", key); + throw ConfigurationError{fmt::format("Missing key \"{}\"", key)}; + } +} + +template bool get_to(const json &j, const std::string &key, T &out) { + try { + j.at(key).get_to(out); + return true; + } catch (const std::out_of_range &) { + fmt::print(fg(fmt::color::red), "Missing key \"{}\"\n", key); + return false; + } catch (const json::type_error &) { + fmt::print(fg(fmt::color::red), "Key \"{}\" is of wrong type\n", key); + return false; + } +} + +template +bool get_to(const json &j, const std::string &key, T &out, const std::string &error_msg) { + if (!get_to(j, key, out)) { + fmt::print(fg(fmt::color::red), error_msg); + return false; + } + + return true; +} + +template bool get_to(const json &j, const std::string &key, T &out, bool &success) { + const bool ret = get_to(j, key, out); + if (!ret) { + success = false; + } + return ret; +} + +template +bool get_to(const json &j, const std::string &key, T &out, const std::string &error_msg, + bool &success) { + const bool ret = get_to(j, key, out, error_msg); + if (!ret) { + success = false; + } + return ret; +} + +void rebase_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { + if (path.is_relative()) { + path = std::filesystem::weakly_canonical(base_dir / path); + } + + if (!std::filesystem::exists(path)) { + throw ConfigurationError{fmt::format("Path does not exist: {}", path.string())}; + } +} + +/// @brief Get a path, based on base_dir, and check if it exists +/// @param j Input JSON +/// @param base_dir Base folder +/// @return An absolute path, assuming that base_dir is the base if relative +/// @throw json::type_error: Invalid JSON types +/// @throw ConfigurationError: Path does not exist +std::filesystem::path get_valid_path(const json &j, const std::filesystem::path &base_dir) { + auto path = j.get(); + rebase_path(path, base_dir); + return path; +} + +bool get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, + std::filesystem::path &out) { + if (!get_to(j, key, out)) { + return false; + } + + try { + rebase_path(out, base_dir); + } catch (const ConfigurationError &) { + fmt::print(fg(fmt::color::red), "Could not find file {}", out.string()); + return false; + } + + return true; +} + +void get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, + std::filesystem::path &out, bool &success) { + if (!get_valid_path_to(j, key, base_dir, out)) { + success = false; + } +} + +/// @brief Load FileInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return FileInfo +/// @throw ConfigurationError: Invalid config file format +auto get_file_info(const json &j, const std::filesystem::path &base_dir) { + const auto dataset = get_key(j, "dataset"); + + bool success = true; + poco::FileInfo info; + get_valid_path_to(dataset, "name", base_dir, info.name, success); + get_to(dataset, "format", info.format, success); + get_to(dataset, "delimiter", info.delimiter, success); + get_to(dataset, "columns", info.columns, success); + if (!success) { + throw ConfigurationError{"Could not load input file info"}; + } + + return info; +} + +/// @brief Load BaselineInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return BaselineInfo +/// @throw json::type_error: Invalid JSON types +/// @throw ConfigurationError: One or more files could not be found +auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { + const auto &adj = get_key(j, "baseline_adjustments"); + + bool success = true; + poco::BaselineInfo info; + get_to(adj, "format", info.format, success); + get_to(adj, "delimiter", info.delimiter, success); + get_to(adj, "encoding", info.encoding, success); + if (get_to(adj, "file_names", info.file_names, success)) { + // Rebase paths and check for errors + for (auto &[name, path] : info.file_names) { + try { + rebase_path(path, base_dir); + fmt::print("{:<14}, file: {}\n", name, path.string()); + } catch (const ConfigurationError &) { + fmt::print(fg(fmt::color::red), "Could not find file: {}\n", path.string()); + success = false; + } + } + } + + if (!success) { + throw ConfigurationError{"Could not get baseline adjustments"}; + } + + return info; +} + +/// @brief Load ModellingInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @throw json::type_error: Invalid JSON types +/// @throw ConfigurationError: Could not load modelling info +void load_modelling_info(const json &j, const std::filesystem::path &base_dir, + Configuration &config) { + bool success = true; + const auto modelling = get_key(j, "modelling"); + + auto &info = config.modelling; + get_to(modelling, "risk_factors", info.risk_factors, success); + + // Rebase paths and check for errors + if (get_to(modelling, "risk_factor_models", info.risk_factor_models, success)) { + for (auto &[type, path] : info.risk_factor_models) { + try { + rebase_path(path, base_dir); + fmt::print("{:<14}, file: {}\n", type, path.string()); + } catch (const ConfigurationError &) { + success = false; + fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", type, + path.string()); + } + } + } + + try { + info.baseline_adjustment = get_baseline_info(modelling, base_dir); + } catch (const std::exception &e) { + success = false; + fmt::print(fmt::fg(fmt::color::red), "Could not load baseline adjustment: {}\n", e.what()); + } + + try { + // SES mapping + // TODO: Maybe this needs its own helper function + config.ses = get_key(modelling, "ses_model").get(); + } catch (const std::exception &e) { + success = false; + fmt::print(fmt::fg(fmt::color::red), "Could not load SES mappings"); + } + + if (!success) { + throw ConfigurationError("Could not load modelling info"); + } +} + +void load_interventions(const json &j, Configuration &config) { + const auto interventions = get_key(j, "interventions"); + + try { + // If the type of intervention is null, then there's nothing else to do + if (interventions.at("active_type_id").is_null()) { + return; + } + } catch (const std::out_of_range &) { + throw ConfigurationError{"Interventions section missing key \"active_type_id\""}; + } + + core::Identifier active_type_id; + try { + active_type_id = interventions["active_type_id"].get(); + } catch (const json::type_error &) { + throw ConfigurationError{"active_type_id key must be of type string"}; + } + + /* + * NB: This loads all of the policy scenario info from the JSON file, which is + * strictly speaking unnecessary, but it does mean that we can verify the data + * format is correct. + */ + std::unordered_map policy_types; + if (!get_to(interventions, "types", policy_types)) { + throw ConfigurationError{"Could not load policy types from interventions section"}; + } + + try { + config.intervention = policy_types.at(active_type_id); + config.intervention.identifier = active_type_id.to_string(); + config.has_active_intervention = true; + } catch (const std::out_of_range &) { + throw ConfigurationError{fmt::format("Unknown active intervention type identifier: {}", + active_type_id.to_string())}; + } +} + +void load_running_info(const json &j, Configuration &config) { + const auto running = get_key(j, "running"); + + bool success = true; + get_to(running, "start_time", config.start_time, success); + get_to(running, "stop_time", config.stop_time, success); + get_to(running, "trial_runs", config.trial_runs, success); + get_to(running, "sync_timeout_ms", config.sync_timeout_ms, success); + get_to(running, "diseases", config.diseases, success); + + // I copied this logic from the old code, but it seems strange to me. Why do we + // store multiple seeds but only use the first? -- Alex + std::vector seeds; + if (get_to(running, "seed", seeds, success) && !seeds.empty()) { + config.custom_seed = seeds[0]; + } + + // Intervention Policy + try { + load_interventions(running, config); + } catch (const ConfigurationError &e) { + success = false; + fmt::print(fmt::fg(fmt::color::red), "Could not load interventions: {}", e.what()); + } + + if (!success) { + throw ConfigurationError{"Could not load running info"}; + } +} + +bool check_version(const json &j) { + int version; + if (!get_to(j, "version", version, "Invalid definition, file must have a schema version")) { + return false; + } + + if (version != 2) { + fmt::print(fg(fmt::color::red), "Configuration schema version: {} mismatch, supported: 2", + version); + return false; + } + + return true; +} + +auto get_settings(const json &j) { + if (!j.contains("settings")) { + fmt::print(fg(fmt::color::red), "\"settings\" key missing"); + throw ConfigurationError{"\"settings\" key missing"}; + } + + return j["settings"].get(); +} + +void load_inputs(const json &j, const std::filesystem::path &config_dir, Configuration &config) { + const auto inputs = get_key(j, "inputs"); + bool success = true; + + // Input dataset file + try { + config.file = get_file_info(inputs, config_dir); + fmt::print("Input dataset file: {}\n", config.file.name.string()); + } catch (const std::exception &e) { + success = false; + fmt::print(fg(fmt::color::red), "Could not load dataset file: {}\n", e.what()); + } + + // Settings + try { + config.settings = get_settings(inputs); + } catch (const std::exception &e) { + success = false; + fmt::print(fg(fmt::color::red), "Could not load settings info"); + } + + if (!success) { + throw ConfigurationError{"Could not load settings info"}; + } +} + +Configuration get_configuration(CommandOptions &options) { MEASURE_FUNCTION(); namespace fs = std::filesystem; using namespace host::poco; + bool success = true; + Configuration config; + config.job_id = options.job_id; + + // verbosity + config.verbosity = core::VerboseMode::none; + if (options.verbose) { + config.verbosity = core::VerboseMode::verbose; + } + std::ifstream ifs(options.config_file, std::ifstream::in); if (!ifs) { throw ConfigurationError( fmt::format("File {} doesn't exist.", options.config_file.string())); } - auto opt = json::parse(ifs); - if (!opt.contains("version")) { - throw ConfigurationError("Invalid definition, file must have a schema version"); - } + const auto opt = [&ifs]() { + try { + return json::parse(ifs); + } catch (const std::exception &e) { + throw ConfigurationError(fmt::format("Could not parse JSON: {}", e.what())); + } + }(); - const auto version = opt["version"].get(); - if (version != 2) { - throw ConfigurationError( - fmt::format("Configuration schema version: {} mismatch, supported: 2", version)); + if (!check_version(opt)) { + success = false; } // application version @@ -272,82 +492,38 @@ Configuration load_configuration(CommandOptions &options) { // Base dir for relative paths const auto config_dir = options.config_file.parent_path(); - bool success = true; // input dataset file try { - config.file = get_file_info(opt["inputs"]["dataset"], config_dir); + load_inputs(opt, config_dir, config); fmt::print("Input dataset file: {}\n", config.file.name.string()); } catch (const std::exception &e) { success = false; fmt::print(fg(fmt::color::red), "Could not load dataset file: {}\n", e.what()); } - // Settings and SES mapping - config.settings = opt["inputs"]["settings"].get(); - config.ses = opt["modelling"]["ses_model"].get(); - // Modelling information try { - config.modelling = get_modelling_info(opt["modelling"], config_dir); + load_modelling_info(opt, config_dir, config); } catch (const std::exception &e) { success = false; fmt::print(fg(fmt::color::red), "Could not load modelling info: {}\n", e.what()); } - if (!success) { - // TODO: Check more things before aborting - throw std::runtime_error("Could not load config"); - } - - // Run-time - opt["running"]["start_time"].get_to(config.start_time); - opt["running"]["stop_time"].get_to(config.stop_time); - opt["running"]["trial_runs"].get_to(config.trial_runs); - opt["running"]["sync_timeout_ms"].get_to(config.sync_timeout_ms); - auto seed = opt["running"]["seed"].get>(); - if (seed.size() > 0) { - config.custom_seed = seed[0]; - } - - opt["running"]["diseases"].get_to(config.diseases); - - // Intervention Policy - auto &interventions = opt["running"]["interventions"]; - if (!interventions["active_type_id"].is_null()) { - auto active_type = interventions["active_type_id"].get(); - auto &policy_types = interventions["types"]; - for (auto it = policy_types.begin(); it != policy_types.end(); ++it) { - if (core::case_insensitive::equals(it.key(), active_type)) { - config.intervention = it.value().get(); - config.intervention.identifier = core::to_lower(it.key()); - config.has_active_intervention = true; - break; - } - } - - if (!core::case_insensitive::equals(config.intervention.identifier, active_type)) { - throw std::runtime_error( - fmt::format("Unknown active intervention type identifier: {}", active_type)); - } + // Run-time info + try { + load_running_info(opt, config); + } catch (const std::exception &e) { + success = false; + fmt::print(fg(fmt::color::red), "Could not load running info: {}\n", e.what()); } - config.job_id = options.job_id; - config.output = opt["output"].get(); - config.output.folder = expand_environment_variables(config.output.folder); - if (!fs::exists(config.output.folder)) { - fmt::print(fg(fmt::color::dark_salmon), "\nCreating output folder: {} ...\n", - config.output.folder); - if (!std::filesystem::create_directories(config.output.folder)) { - throw std::runtime_error( - fmt::format("Failed to create output folder: {}", config.output.folder)); - } + if (get_to(opt, "output", config.output, "Could not load output info", success)) { + config.output.folder = expand_environment_variables(config.output.folder); } - // verbosity - config.verbosity = core::VerboseMode::none; - if (options.verbose) { - config.verbosity = core::VerboseMode::verbose; + if (!success) { + throw ConfigurationError{"Error loading config file"}; } return config; diff --git a/src/HealthGPS.Console/configuration.h b/src/HealthGPS.Console/configuration.h index d2d7854d3..690890175 100644 --- a/src/HealthGPS.Console/configuration.h +++ b/src/HealthGPS.Console/configuration.h @@ -42,7 +42,7 @@ CommandOptions parse_arguments(cxxopts::Options &options, int &argc, char *argv[ /// @brief Loads the input configuration file, *.json, information /// @param options User command-line options /// @return The configuration file information -Configuration load_configuration(CommandOptions &options); +Configuration get_configuration(CommandOptions &options); /// @brief Gets the collection of diseases that matches the selected input list /// @param data_api The back-end data store instance to be used. diff --git a/src/HealthGPS.Console/program.cpp b/src/HealthGPS.Console/program.cpp index e1693b540..4ad4b6cea 100644 --- a/src/HealthGPS.Console/program.cpp +++ b/src/HealthGPS.Console/program.cpp @@ -40,12 +40,23 @@ int main(int argc, char *argv[]) { // NOLINT(bugprone-exception-escape) // Parse inputs configuration file, *.json. Configuration config; try { - config = load_configuration(cmd_args); + config = get_configuration(cmd_args); } catch (const std::exception &ex) { fmt::print(fg(fmt::color::red), "\n\nInvalid configuration - {}.\n", ex.what()); return exit_application(EXIT_FAILURE); } + // Create output folder + if (!std::filesystem::exists(config.output.folder)) { + fmt::print(fg(fmt::color::dark_salmon), "\nCreating output folder: {} ...\n", + config.output.folder); + if (!std::filesystem::create_directories(config.output.folder)) { + fmt::print(fg(fmt::color::red), "Failed to create output folder: {}\n", + config.output.folder); + return exit_application(EXIT_FAILURE); + } + } + // Load input data file into a datatable asynchronous auto input_table = core::DataTable(); auto table_future = From f480522833fe7294dafd493b10cded67581d0b44 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 10:44:57 +0100 Subject: [PATCH 10/46] Remove superfluous helper functions --- src/HealthGPS.Console/configuration.cpp | 27 +++++-------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index e76d53ebd..1b83aba30 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -157,16 +157,6 @@ template bool get_to(const json &j, const std::string &key, T &out) { } } -template -bool get_to(const json &j, const std::string &key, T &out, const std::string &error_msg) { - if (!get_to(j, key, out)) { - fmt::print(fg(fmt::color::red), error_msg); - return false; - } - - return true; -} - template bool get_to(const json &j, const std::string &key, T &out, bool &success) { const bool ret = get_to(j, key, out); if (!ret) { @@ -175,16 +165,6 @@ template bool get_to(const json &j, const std::string &key, T &out, bo return ret; } -template -bool get_to(const json &j, const std::string &key, T &out, const std::string &error_msg, - bool &success) { - const bool ret = get_to(j, key, out, error_msg); - if (!ret) { - success = false; - } - return ret; -} - void rebase_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { if (path.is_relative()) { path = std::filesystem::weakly_canonical(base_dir / path); @@ -404,7 +384,8 @@ void load_running_info(const json &j, Configuration &config) { bool check_version(const json &j) { int version; - if (!get_to(j, "version", version, "Invalid definition, file must have a schema version")) { + if (!get_to(j, "version", version)) { + fmt::print(fg(fmt::color::red), "Invalid definition, file must have a schema version"); return false; } @@ -518,8 +499,10 @@ Configuration get_configuration(CommandOptions &options) { fmt::print(fg(fmt::color::red), "Could not load running info: {}\n", e.what()); } - if (get_to(opt, "output", config.output, "Could not load output info", success)) { + if (get_to(opt, "output", config.output, success)) { config.output.folder = expand_environment_variables(config.output.folder); + } else { + fmt::print(fg(fmt::color::red), "Could not load output info"); } if (!success) { From 2597d3269f9aa0f7ccef9545f8713f3ad29e23a1 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 13:20:25 +0100 Subject: [PATCH 11/46] Fix: Seemingly MSVC isn't happy about casting filesystem::paths to strings --- src/HealthGPS.Console/model_parser.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/HealthGPS.Console/model_parser.cpp b/src/HealthGPS.Console/model_parser.cpp index be59d0706..490a6abbf 100644 --- a/src/HealthGPS.Console/model_parser.cpp +++ b/src/HealthGPS.Console/model_parser.cpp @@ -20,11 +20,10 @@ namespace host { hgps::BaselineAdjustment load_baseline_adjustments(const poco::BaselineInfo &info) { MEASURE_FUNCTION(); - auto &male_filename = info.file_names.at("factorsmean_male"); - auto &female_filename = info.file_names.at("factorsmean_female"); + const auto male_filename = info.file_names.at("factorsmean_male").string(); + const auto female_filename = info.file_names.at("factorsmean_female").string(); try { - if (hgps::core::case_insensitive::equals(info.format, "CSV")) { auto data = std::map>>{}; @@ -38,7 +37,7 @@ hgps::BaselineAdjustment load_baseline_adjustments(const poco::BaselineInfo &inf } } catch (const std::exception &ex) { fmt::print(fg(fmt::color::red), "Failed to parse adjustment file: {} or {}. {}\n", - male_filename.string(), female_filename.string(), ex.what()); + male_filename, female_filename, ex.what()); throw; } } From 9a1188cc106a8076de54b9a8dda6883292a17060 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 11:02:22 +0100 Subject: [PATCH 12/46] Move some functions out of configuration.cpp to program.cpp They make more sense there. --- src/HealthGPS.Console/configuration.cpp | 12 ---------- src/HealthGPS.Console/configuration.h | 7 ------ src/HealthGPS.Console/program.cpp | 32 ++++++++++++++++++------- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index 1b83aba30..fff3f3261 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -33,11 +33,6 @@ namespace host { using namespace hgps; using json = nlohmann::json; -std::string get_time_now_str() { - auto tp = std::chrono::system_clock::now(); - return fmt::format("{0:%F %H:%M:}{1:%S} {0:%Z}", tp, tp.time_since_epoch()); -} - cxxopts::Options create_options() { cxxopts::Options options("HealthGPS.Console", "Health-GPS microsimulation for policy options."); options.add_options()("f,file", "Configuration file full name.", cxxopts::value())( @@ -50,13 +45,6 @@ cxxopts::Options create_options() { return options; } -void print_app_title() { - fmt::print(fg(fmt::color::yellow) | fmt::emphasis::bold, - "\n# Health-GPS Microsimulation for Policy Options #\n\n"); - - fmt::print("Today: {}\n\n", get_time_now_str()); -} - CommandOptions parse_arguments(cxxopts::Options &options, int &argc, char *argv[]) { MEASURE_FUNCTION(); namespace fs = std::filesystem; diff --git a/src/HealthGPS.Console/configuration.h b/src/HealthGPS.Console/configuration.h index 690890175..d76f8f61b 100644 --- a/src/HealthGPS.Console/configuration.h +++ b/src/HealthGPS.Console/configuration.h @@ -21,17 +21,10 @@ class ConfigurationError : public std::runtime_error { ConfigurationError(const std::string &msg); }; -/// @brief Get a string representation of current system time -/// @return The system time as string -std::string get_time_now_str(); - /// @brief Creates the command-line interface (CLI) options /// @return Health-GPS CLI options cxxopts::Options create_options(); -/// @brief Prints application start-up messages -void print_app_title(); - /// @brief Parses the command-line interface (CLI) arguments /// @param options The valid CLI options /// @param argc Number of input arguments diff --git a/src/HealthGPS.Console/program.cpp b/src/HealthGPS.Console/program.cpp index 4ad4b6cea..6dcbd0252 100644 --- a/src/HealthGPS.Console/program.cpp +++ b/src/HealthGPS.Console/program.cpp @@ -8,12 +8,35 @@ #include "HealthGPS/event_bus.h" #include "event_monitor.h" +#include #include +#include + +/// @brief Get a string representation of current system time +/// @return The system time as string +std::string get_time_now_str() { + auto tp = std::chrono::system_clock::now(); + return fmt::format("{0:%F %H:%M:}{1:%S} {0:%Z}", tp, tp.time_since_epoch()); +} + +/// @brief Prints application start-up messages +void print_app_title() { + fmt::print(fg(fmt::color::yellow) | fmt::emphasis::bold, + "\n# Health-GPS Microsimulation for Policy Options #\n\n"); + + fmt::print("Today: {}\n\n", get_time_now_str()); +} + /// @brief Prints application exit message /// @param exit_code The application exit code /// @return The respective exit code -int exit_application(int exit_code); +int exit_application(int exit_code) { + fmt::print("\n\n"); + fmt::print(fg(fmt::color::yellow) | fmt::emphasis::bold, "Goodbye."); + fmt::print(" {}.\n\n", get_time_now_str()); + return exit_code; +} /// @brief Health-GPS host application entry point /// @param argc The number of command arguments @@ -174,13 +197,6 @@ int main(int argc, char *argv[]) { // NOLINT(bugprone-exception-escape) return exit_application(EXIT_SUCCESS); } -int exit_application(int exit_code) { - fmt::print("\n\n"); - fmt::print(fg(fmt::color::yellow) | fmt::emphasis::bold, "Goodbye."); - fmt::print(" {}.\n\n", host::get_time_now_str()); - return exit_code; -} - /// @brief Top-level namespace for Health-GPS Console host application namespace host { /// @brief Internal details namespace for private data types and functions From 3efc2be2cdeec88446cd333e5ad77b202f7358df Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 11:20:38 +0100 Subject: [PATCH 13/46] Put functionality related to command-line args in own cpp file --- src/HealthGPS.Console/CMakeLists.txt | 3 +- src/HealthGPS.Console/command_options.cpp | 96 +++++++++++++++++++++++ src/HealthGPS.Console/command_options.h | 39 +++++++++ src/HealthGPS.Console/configuration.cpp | 92 ---------------------- src/HealthGPS.Console/configuration.h | 71 ++++++++++++++--- src/HealthGPS.Console/jsonparser.h | 2 +- src/HealthGPS.Console/model_parser.h | 1 - src/HealthGPS.Console/options.h | 81 ------------------- 8 files changed, 196 insertions(+), 189 deletions(-) create mode 100644 src/HealthGPS.Console/command_options.cpp create mode 100644 src/HealthGPS.Console/command_options.h delete mode 100644 src/HealthGPS.Console/options.h diff --git a/src/HealthGPS.Console/CMakeLists.txt b/src/HealthGPS.Console/CMakeLists.txt index 453f882cb..e48f055ec 100644 --- a/src/HealthGPS.Console/CMakeLists.txt +++ b/src/HealthGPS.Console/CMakeLists.txt @@ -13,6 +13,8 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) target_sources(HealthGPS.Console PRIVATE + "command_options.cpp" + "command_options.h" "configuration.cpp" "configuration.h" "csvparser.cpp" @@ -27,7 +29,6 @@ target_sources(HealthGPS.Console "jsonparser.cpp" "jsonparser.h" "model_info.h" - "options.h" "resource.h" "result_writer.h" "riskmodel.h" diff --git a/src/HealthGPS.Console/command_options.cpp b/src/HealthGPS.Console/command_options.cpp new file mode 100644 index 000000000..c891c0b85 --- /dev/null +++ b/src/HealthGPS.Console/command_options.cpp @@ -0,0 +1,96 @@ +#include "command_options.h" +#include "version.h" + +#include + +#include + +namespace host { + +cxxopts::Options create_options() { + cxxopts::Options options("HealthGPS.Console", "Health-GPS microsimulation for policy options."); + options.add_options()("f,file", "Configuration file full name.", cxxopts::value())( + "s,storage", "Path to root folder of the data storage.", cxxopts::value())( + "j,jobid", "The batch execution job identifier.", + cxxopts::value())("verbose", "Print more information about progress", + cxxopts::value()->default_value("false"))( + "help", "Help about this application.")("version", "Print the application version number."); + + return options; +} + +CommandOptions parse_arguments(cxxopts::Options &options, int &argc, char *argv[]) { + namespace fs = std::filesystem; + + CommandOptions cmd; + try { + cmd.success = true; + cmd.exit_code = EXIT_SUCCESS; + cmd.verbose = false; + auto result = options.parse(argc, argv); + if (result.count("help")) { + std::cout << options.help() << std::endl; + cmd.success = false; + return cmd; + } + + if (result.count("version")) { + fmt::print("Version {}\n\n", PROJECT_VERSION); + cmd.success = false; + return cmd; + } + + cmd.verbose = result["verbose"].as(); + if (cmd.verbose) { + fmt::print(fg(fmt::color::dark_salmon), "Verbose output enabled\n"); + } + + if (result.count("file")) { + cmd.config_file = result["file"].as(); + if (cmd.config_file.is_relative()) { + cmd.config_file = std::filesystem::absolute(cmd.config_file); + fmt::print("Configuration file..: {}\n", cmd.config_file.string()); + } + } + + if (!fs::exists(cmd.config_file)) { + fmt::print(fg(fmt::color::red), "\nConfiguration file: {} not found.\n", + cmd.config_file.string()); + cmd.exit_code = EXIT_FAILURE; + } + + if (result.count("storage")) { + cmd.storage_folder = result["storage"].as(); + if (cmd.storage_folder.is_relative()) { + cmd.storage_folder = std::filesystem::absolute(cmd.storage_folder); + fmt::print("File storage folder.: {}\n", cmd.storage_folder.string()); + } + } + + if (!fs::exists(cmd.storage_folder)) { + fmt::print(fg(fmt::color::red), "\nFile storage folder: {} not found.\n", + cmd.storage_folder.string()); + cmd.exit_code = EXIT_FAILURE; + } + + if (result.count("jobid")) { + cmd.job_id = result["jobid"].as(); + if (cmd.job_id < 1) { + fmt::print(fg(fmt::color::red), + "\nJob identifier value outside range: (0 < x) given: {}.\n", + std::to_string(cmd.job_id)); + cmd.exit_code = EXIT_FAILURE; + } + } + + cmd.success = cmd.exit_code == EXIT_SUCCESS; + } catch (const cxxopts::exceptions::exception &ex) { + fmt::print(fg(fmt::color::red), "\nInvalid command line argument: {}.\n", ex.what()); + fmt::print("\n{}\n", options.help()); + cmd.success = false; + cmd.exit_code = EXIT_FAILURE; + } + + return cmd; +} +} // namespace host \ No newline at end of file diff --git a/src/HealthGPS.Console/command_options.h b/src/HealthGPS.Console/command_options.h new file mode 100644 index 000000000..83d1efb30 --- /dev/null +++ b/src/HealthGPS.Console/command_options.h @@ -0,0 +1,39 @@ +#pragma once +#include + +#include + +namespace host { +/// @brief Defines the Command Line Interface (CLI) arguments options +struct CommandOptions { + /// @brief Indicates whether the argument parsing succeed + bool success{}; + + /// @brief The exit code to return, in case of CLI arguments parsing failure + int exit_code{}; + + /// @brief The configuration file argument value + std::filesystem::path config_file{}; + + /// @brief The back-end storage full path argument value + std::filesystem::path storage_folder{}; + + /// @brief Indicates whether the application logging is verbose + bool verbose{}; + + /// @brief The batch job identifier value, optional. + int job_id{}; +}; + +/// @brief Creates the command-line interface (CLI) options +/// @return Health-GPS CLI options +cxxopts::Options create_options(); + +/// @brief Parses the command-line interface (CLI) arguments +/// @param options The valid CLI options +/// @param argc Number of input arguments +/// @param argv List of input arguments +/// @return User command-line options +CommandOptions parse_arguments(cxxopts::Options &options, int &argc, char *argv[]); + +} // namespace host \ No newline at end of file diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index fff3f3261..802b31e71 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -33,94 +33,6 @@ namespace host { using namespace hgps; using json = nlohmann::json; -cxxopts::Options create_options() { - cxxopts::Options options("HealthGPS.Console", "Health-GPS microsimulation for policy options."); - options.add_options()("f,file", "Configuration file full name.", cxxopts::value())( - "s,storage", "Path to root folder of the data storage.", cxxopts::value())( - "j,jobid", "The batch execution job identifier.", - cxxopts::value())("verbose", "Print more information about progress", - cxxopts::value()->default_value("false"))( - "help", "Help about this application.")("version", "Print the application version number."); - - return options; -} - -CommandOptions parse_arguments(cxxopts::Options &options, int &argc, char *argv[]) { - MEASURE_FUNCTION(); - namespace fs = std::filesystem; - - CommandOptions cmd; - try { - cmd.success = true; - cmd.exit_code = EXIT_SUCCESS; - cmd.verbose = false; - auto result = options.parse(argc, argv); - if (result.count("help")) { - std::cout << options.help() << std::endl; - cmd.success = false; - return cmd; - } - - if (result.count("version")) { - fmt::print("Version {}\n\n", PROJECT_VERSION); - cmd.success = false; - return cmd; - } - - cmd.verbose = result["verbose"].as(); - if (cmd.verbose) { - fmt::print(fg(fmt::color::dark_salmon), "Verbose output enabled\n"); - } - - if (result.count("file")) { - cmd.config_file = result["file"].as(); - if (cmd.config_file.is_relative()) { - cmd.config_file = std::filesystem::absolute(cmd.config_file); - fmt::print("Configuration file..: {}\n", cmd.config_file.string()); - } - } - - if (!fs::exists(cmd.config_file)) { - fmt::print(fg(fmt::color::red), "\nConfiguration file: {} not found.\n", - cmd.config_file.string()); - cmd.exit_code = EXIT_FAILURE; - } - - if (result.count("storage")) { - cmd.storage_folder = result["storage"].as(); - if (cmd.storage_folder.is_relative()) { - cmd.storage_folder = std::filesystem::absolute(cmd.storage_folder); - fmt::print("File storage folder.: {}\n", cmd.storage_folder.string()); - } - } - - if (!fs::exists(cmd.storage_folder)) { - fmt::print(fg(fmt::color::red), "\nFile storage folder: {} not found.\n", - cmd.storage_folder.string()); - cmd.exit_code = EXIT_FAILURE; - } - - if (result.count("jobid")) { - cmd.job_id = result["jobid"].as(); - if (cmd.job_id < 1) { - fmt::print(fg(fmt::color::red), - "\nJob identifier value outside range: (0 < x) given: {}.\n", - std::to_string(cmd.job_id)); - cmd.exit_code = EXIT_FAILURE; - } - } - - cmd.success = cmd.exit_code == EXIT_SUCCESS; - } catch (const cxxopts::exceptions::exception &ex) { - fmt::print(fg(fmt::color::red), "\nInvalid command line argument: {}.\n", ex.what()); - fmt::print("\n{}\n", options.help()); - cmd.success = false; - cmd.exit_code = EXIT_FAILURE; - } - - return cmd; -} - ConfigurationError::ConfigurationError(const std::string &msg) : std::runtime_error{msg} {} auto get_key(const json &j, const std::string &key) { @@ -455,10 +367,6 @@ Configuration get_configuration(CommandOptions &options) { success = false; } - // application version - config.app_name = PROJECT_NAME; - config.app_version = PROJECT_VERSION; - // Base dir for relative paths const auto config_dir = options.config_file.parent_path(); diff --git a/src/HealthGPS.Console/configuration.h b/src/HealthGPS.Console/configuration.h index d76f8f61b..13cc9dd7f 100644 --- a/src/HealthGPS.Console/configuration.h +++ b/src/HealthGPS.Console/configuration.h @@ -1,5 +1,8 @@ #pragma once -#include + +#include "command_options.h" +#include "poco.h" +#include "version.h" #include "HealthGPS/healthgps.h" #include "HealthGPS/intervention_scenario.h" @@ -8,30 +11,72 @@ #include "HealthGPS.Core/api.h" -#include "options.h" #include "result_file_writer.h" #include namespace host { +/// @brief Defines the application configuration data structure +struct Configuration { + /// @brief The input data file details + poco::FileInfo file; + + /// @brief Experiment population settings + poco::SettingsInfo settings; + + /// @brief Socio-economic status (SES) model inputs + poco::SESInfo ses; + + /// @brief User defined model and parameters information + poco::ModellingInfo modelling; + + /// @brief List of diseases to include in experiment + std::vector diseases; + + /// @brief Simulation initialisation custom seed value, optional + std::optional custom_seed; + + /// @brief The experiment start time (simulation clock) + unsigned int start_time{}; + + /// @brief The experiment stop time (simulation clock) + unsigned int stop_time{}; + + /// @brief The number of simulation runs (replications) to execute + unsigned int trial_runs{}; + + /// @brief Baseline to intervention data synchronisation time out (milliseconds) + unsigned int sync_timeout_ms{}; + + /// @brief Indicates whether an alternative intervention policy is active + bool has_active_intervention{false}; + + /// @brief The active intervention policy definition + poco::PolicyScenarioInfo intervention; + + /// @brief Experiment output folder and file information + poco::OutputInfo output; + + /// @brief Application logging verbosity mode + hgps::core::VerboseMode verbosity{}; + + /// @brief Experiment batch job identifier + int job_id{}; + + /// @brief Experiment model name + const char *app_name = PROJECT_NAME; + + /// @brief Experiment model version + const char *app_version = PROJECT_VERSION; +}; + /// @brief Represents an error that occurred with the format of a config file class ConfigurationError : public std::runtime_error { public: ConfigurationError(const std::string &msg); }; -/// @brief Creates the command-line interface (CLI) options -/// @return Health-GPS CLI options -cxxopts::Options create_options(); - -/// @brief Parses the command-line interface (CLI) arguments -/// @param options The valid CLI options -/// @param argc Number of input arguments -/// @param argv List of input arguments -/// @return User command-line options -CommandOptions parse_arguments(cxxopts::Options &options, int &argc, char *argv[]); - /// @brief Loads the input configuration file, *.json, information /// @param options User command-line options /// @return The configuration file information diff --git a/src/HealthGPS.Console/jsonparser.h b/src/HealthGPS.Console/jsonparser.h index 041ee217f..9139db436 100644 --- a/src/HealthGPS.Console/jsonparser.h +++ b/src/HealthGPS.Console/jsonparser.h @@ -1,5 +1,5 @@ #pragma once -#include "options.h" +#include "poco.h" #include "riskmodel.h" #include diff --git a/src/HealthGPS.Console/model_parser.h b/src/HealthGPS.Console/model_parser.h index 21f954d4f..27f331fb3 100644 --- a/src/HealthGPS.Console/model_parser.h +++ b/src/HealthGPS.Console/model_parser.h @@ -3,7 +3,6 @@ #include "HealthGPS/riskfactor_adjustment_types.h" #include "jsonparser.h" -#include "options.h" #include diff --git a/src/HealthGPS.Console/options.h b/src/HealthGPS.Console/options.h deleted file mode 100644 index cbf4f7c75..000000000 --- a/src/HealthGPS.Console/options.h +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once -#include "HealthGPS.Core/forward_type.h" -#include "poco.h" -#include - -namespace host { -/// @brief Defines the Command Line Interface (CLI) arguments options -struct CommandOptions { - /// @brief Indicates whether the argument parsing succeed - bool success{}; - - /// @brief The exit code to return, in case of CLI arguments parsing failure - int exit_code{}; - - /// @brief The configuration file argument value - std::filesystem::path config_file{}; - - /// @brief The back-end storage full path argument value - std::filesystem::path storage_folder{}; - - /// @brief Indicates whether the application logging is verbose - bool verbose{}; - - /// @brief The batch job identifier value, optional. - int job_id{}; -}; - -/// @brief Defines the application configuration data structure -struct Configuration { - /// @brief The input data file details - poco::FileInfo file; - - /// @brief Experiment population settings - poco::SettingsInfo settings; - - /// @brief Socio-economic status (SES) model inputs - poco::SESInfo ses; - - /// @brief User defined model and parameters information - poco::ModellingInfo modelling; - - /// @brief List of diseases to include in experiment - std::vector diseases; - - /// @brief Simulation initialisation custom seed value, optional - std::optional custom_seed; - - /// @brief The experiment start time (simulation clock) - unsigned int start_time{}; - - /// @brief The experiment stop time (simulation clock) - unsigned int stop_time{}; - - /// @brief The number of simulation runs (replications) to execute - unsigned int trial_runs{}; - - /// @brief Baseline to intervention data synchronisation time out (milliseconds) - unsigned int sync_timeout_ms{}; - - /// @brief Indicates whether an alternative intervention policy is active - bool has_active_intervention{false}; - - /// @brief The active intervention policy definition - poco::PolicyScenarioInfo intervention; - - /// @brief Experiment output folder and file information - poco::OutputInfo output; - - /// @brief Application logging verbosity mode - hgps::core::VerboseMode verbosity{}; - - /// @brief Experiment batch job identifier - int job_id{}; - - /// @brief Experiment model name - std::string app_name; - - /// @brief Experiment model version - std::string app_version; -}; -} // namespace host \ No newline at end of file From 0d1f803baef8f70c2f43b0f4cdcdf2574759f7c4 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 12:16:53 +0100 Subject: [PATCH 14/46] Move some parsing functions into configuration_parsing.cpp --- src/HealthGPS.Console/CMakeLists.txt | 2 + src/HealthGPS.Console/configuration.cpp | 315 +--------------- .../configuration_parsing.cpp | 337 ++++++++++++++++++ src/HealthGPS.Console/configuration_parsing.h | 37 ++ 4 files changed, 387 insertions(+), 304 deletions(-) create mode 100644 src/HealthGPS.Console/configuration_parsing.cpp create mode 100644 src/HealthGPS.Console/configuration_parsing.h diff --git a/src/HealthGPS.Console/CMakeLists.txt b/src/HealthGPS.Console/CMakeLists.txt index e48f055ec..ffe2027e8 100644 --- a/src/HealthGPS.Console/CMakeLists.txt +++ b/src/HealthGPS.Console/CMakeLists.txt @@ -17,6 +17,8 @@ target_sources(HealthGPS.Console "command_options.h" "configuration.cpp" "configuration.h" + "configuration_parsing.cpp" + "configuration_parsing.h" "csvparser.cpp" "csvparser.h" "event_monitor.cpp" diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index 802b31e71..b918e57c3 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -1,4 +1,5 @@ #include "configuration.h" +#include "configuration_parsing.h" #include "jsonparser.h" #include "version.h" @@ -35,304 +36,6 @@ using json = nlohmann::json; ConfigurationError::ConfigurationError(const std::string &msg) : std::runtime_error{msg} {} -auto get_key(const json &j, const std::string &key) { - try { - return j.at(key); - } catch (const std::out_of_range &) { - fmt::print(fg(fmt::color::red), "Missing key \"{}\"\n", key); - throw ConfigurationError{fmt::format("Missing key \"{}\"", key)}; - } -} - -template bool get_to(const json &j, const std::string &key, T &out) { - try { - j.at(key).get_to(out); - return true; - } catch (const std::out_of_range &) { - fmt::print(fg(fmt::color::red), "Missing key \"{}\"\n", key); - return false; - } catch (const json::type_error &) { - fmt::print(fg(fmt::color::red), "Key \"{}\" is of wrong type\n", key); - return false; - } -} - -template bool get_to(const json &j, const std::string &key, T &out, bool &success) { - const bool ret = get_to(j, key, out); - if (!ret) { - success = false; - } - return ret; -} - -void rebase_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { - if (path.is_relative()) { - path = std::filesystem::weakly_canonical(base_dir / path); - } - - if (!std::filesystem::exists(path)) { - throw ConfigurationError{fmt::format("Path does not exist: {}", path.string())}; - } -} - -/// @brief Get a path, based on base_dir, and check if it exists -/// @param j Input JSON -/// @param base_dir Base folder -/// @return An absolute path, assuming that base_dir is the base if relative -/// @throw json::type_error: Invalid JSON types -/// @throw ConfigurationError: Path does not exist -std::filesystem::path get_valid_path(const json &j, const std::filesystem::path &base_dir) { - auto path = j.get(); - rebase_path(path, base_dir); - return path; -} - -bool get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, - std::filesystem::path &out) { - if (!get_to(j, key, out)) { - return false; - } - - try { - rebase_path(out, base_dir); - } catch (const ConfigurationError &) { - fmt::print(fg(fmt::color::red), "Could not find file {}", out.string()); - return false; - } - - return true; -} - -void get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, - std::filesystem::path &out, bool &success) { - if (!get_valid_path_to(j, key, base_dir, out)) { - success = false; - } -} - -/// @brief Load FileInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return FileInfo -/// @throw ConfigurationError: Invalid config file format -auto get_file_info(const json &j, const std::filesystem::path &base_dir) { - const auto dataset = get_key(j, "dataset"); - - bool success = true; - poco::FileInfo info; - get_valid_path_to(dataset, "name", base_dir, info.name, success); - get_to(dataset, "format", info.format, success); - get_to(dataset, "delimiter", info.delimiter, success); - get_to(dataset, "columns", info.columns, success); - if (!success) { - throw ConfigurationError{"Could not load input file info"}; - } - - return info; -} - -/// @brief Load BaselineInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return BaselineInfo -/// @throw json::type_error: Invalid JSON types -/// @throw ConfigurationError: One or more files could not be found -auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { - const auto &adj = get_key(j, "baseline_adjustments"); - - bool success = true; - poco::BaselineInfo info; - get_to(adj, "format", info.format, success); - get_to(adj, "delimiter", info.delimiter, success); - get_to(adj, "encoding", info.encoding, success); - if (get_to(adj, "file_names", info.file_names, success)) { - // Rebase paths and check for errors - for (auto &[name, path] : info.file_names) { - try { - rebase_path(path, base_dir); - fmt::print("{:<14}, file: {}\n", name, path.string()); - } catch (const ConfigurationError &) { - fmt::print(fg(fmt::color::red), "Could not find file: {}\n", path.string()); - success = false; - } - } - } - - if (!success) { - throw ConfigurationError{"Could not get baseline adjustments"}; - } - - return info; -} - -/// @brief Load ModellingInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @throw json::type_error: Invalid JSON types -/// @throw ConfigurationError: Could not load modelling info -void load_modelling_info(const json &j, const std::filesystem::path &base_dir, - Configuration &config) { - bool success = true; - const auto modelling = get_key(j, "modelling"); - - auto &info = config.modelling; - get_to(modelling, "risk_factors", info.risk_factors, success); - - // Rebase paths and check for errors - if (get_to(modelling, "risk_factor_models", info.risk_factor_models, success)) { - for (auto &[type, path] : info.risk_factor_models) { - try { - rebase_path(path, base_dir); - fmt::print("{:<14}, file: {}\n", type, path.string()); - } catch (const ConfigurationError &) { - success = false; - fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", type, - path.string()); - } - } - } - - try { - info.baseline_adjustment = get_baseline_info(modelling, base_dir); - } catch (const std::exception &e) { - success = false; - fmt::print(fmt::fg(fmt::color::red), "Could not load baseline adjustment: {}\n", e.what()); - } - - try { - // SES mapping - // TODO: Maybe this needs its own helper function - config.ses = get_key(modelling, "ses_model").get(); - } catch (const std::exception &e) { - success = false; - fmt::print(fmt::fg(fmt::color::red), "Could not load SES mappings"); - } - - if (!success) { - throw ConfigurationError("Could not load modelling info"); - } -} - -void load_interventions(const json &j, Configuration &config) { - const auto interventions = get_key(j, "interventions"); - - try { - // If the type of intervention is null, then there's nothing else to do - if (interventions.at("active_type_id").is_null()) { - return; - } - } catch (const std::out_of_range &) { - throw ConfigurationError{"Interventions section missing key \"active_type_id\""}; - } - - core::Identifier active_type_id; - try { - active_type_id = interventions["active_type_id"].get(); - } catch (const json::type_error &) { - throw ConfigurationError{"active_type_id key must be of type string"}; - } - - /* - * NB: This loads all of the policy scenario info from the JSON file, which is - * strictly speaking unnecessary, but it does mean that we can verify the data - * format is correct. - */ - std::unordered_map policy_types; - if (!get_to(interventions, "types", policy_types)) { - throw ConfigurationError{"Could not load policy types from interventions section"}; - } - - try { - config.intervention = policy_types.at(active_type_id); - config.intervention.identifier = active_type_id.to_string(); - config.has_active_intervention = true; - } catch (const std::out_of_range &) { - throw ConfigurationError{fmt::format("Unknown active intervention type identifier: {}", - active_type_id.to_string())}; - } -} - -void load_running_info(const json &j, Configuration &config) { - const auto running = get_key(j, "running"); - - bool success = true; - get_to(running, "start_time", config.start_time, success); - get_to(running, "stop_time", config.stop_time, success); - get_to(running, "trial_runs", config.trial_runs, success); - get_to(running, "sync_timeout_ms", config.sync_timeout_ms, success); - get_to(running, "diseases", config.diseases, success); - - // I copied this logic from the old code, but it seems strange to me. Why do we - // store multiple seeds but only use the first? -- Alex - std::vector seeds; - if (get_to(running, "seed", seeds, success) && !seeds.empty()) { - config.custom_seed = seeds[0]; - } - - // Intervention Policy - try { - load_interventions(running, config); - } catch (const ConfigurationError &e) { - success = false; - fmt::print(fmt::fg(fmt::color::red), "Could not load interventions: {}", e.what()); - } - - if (!success) { - throw ConfigurationError{"Could not load running info"}; - } -} - -bool check_version(const json &j) { - int version; - if (!get_to(j, "version", version)) { - fmt::print(fg(fmt::color::red), "Invalid definition, file must have a schema version"); - return false; - } - - if (version != 2) { - fmt::print(fg(fmt::color::red), "Configuration schema version: {} mismatch, supported: 2", - version); - return false; - } - - return true; -} - -auto get_settings(const json &j) { - if (!j.contains("settings")) { - fmt::print(fg(fmt::color::red), "\"settings\" key missing"); - throw ConfigurationError{"\"settings\" key missing"}; - } - - return j["settings"].get(); -} - -void load_inputs(const json &j, const std::filesystem::path &config_dir, Configuration &config) { - const auto inputs = get_key(j, "inputs"); - bool success = true; - - // Input dataset file - try { - config.file = get_file_info(inputs, config_dir); - fmt::print("Input dataset file: {}\n", config.file.name.string()); - } catch (const std::exception &e) { - success = false; - fmt::print(fg(fmt::color::red), "Could not load dataset file: {}\n", e.what()); - } - - // Settings - try { - config.settings = get_settings(inputs); - } catch (const std::exception &e) { - success = false; - fmt::print(fg(fmt::color::red), "Could not load settings info"); - } - - if (!success) { - throw ConfigurationError{"Could not load settings info"}; - } -} - Configuration get_configuration(CommandOptions &options) { MEASURE_FUNCTION(); namespace fs = std::filesystem; @@ -363,7 +66,10 @@ Configuration get_configuration(CommandOptions &options) { } }(); - if (!check_version(opt)) { + // Check the file format version + try { + check_version(opt); + } catch (const ConfigurationError &) { success = false; } @@ -372,7 +78,7 @@ Configuration get_configuration(CommandOptions &options) { // input dataset file try { - load_inputs(opt, config_dir, config); + load_input_info(opt, config, config_dir); fmt::print("Input dataset file: {}\n", config.file.name.string()); } catch (const std::exception &e) { success = false; @@ -381,7 +87,7 @@ Configuration get_configuration(CommandOptions &options) { // Modelling information try { - load_modelling_info(opt, config_dir, config); + load_modelling_info(opt, config, config_dir); } catch (const std::exception &e) { success = false; fmt::print(fg(fmt::color::red), "Could not load modelling info: {}\n", e.what()); @@ -395,9 +101,10 @@ Configuration get_configuration(CommandOptions &options) { fmt::print(fg(fmt::color::red), "Could not load running info: {}\n", e.what()); } - if (get_to(opt, "output", config.output, success)) { - config.output.folder = expand_environment_variables(config.output.folder); - } else { + try { + load_output_info(opt, config); + } catch (const ConfigurationError &) { + success = false; fmt::print(fg(fmt::color::red), "Could not load output info"); } diff --git a/src/HealthGPS.Console/configuration_parsing.cpp b/src/HealthGPS.Console/configuration_parsing.cpp new file mode 100644 index 000000000..1a36ff947 --- /dev/null +++ b/src/HealthGPS.Console/configuration_parsing.cpp @@ -0,0 +1,337 @@ +#include "configuration_parsing.h" +#include "jsonparser.h" + +#include +#include +#include + +namespace host { +using json = nlohmann::json; + +/// @brief Load value from JSON, printing an error message if it fails +/// @param j JSON object +/// @param key Key to value +/// @throw ConfigurationError: Key not found +/// @return Key value +auto get(const json &j, const std::string &key) { + try { + return j.at(key); + } catch (const std::out_of_range &) { + fmt::print(fg(fmt::color::red), "Missing key \"{}\"\n", key); + throw ConfigurationError{fmt::format("Missing key \"{}\"", key)}; + } +} + +/// @brief Get value from JSON object and store in out +/// @tparam T Type of output object +/// @param j JSON object +/// @param key Key to value +/// @param out Output object +/// @return True if value was retrieved successfully, false otherwise +template bool get_to(const json &j, const std::string &key, T &out) { + try { + j.at(key).get_to(out); + return true; + } catch (const std::out_of_range &) { + fmt::print(fg(fmt::color::red), "Missing key \"{}\"\n", key); + return false; + } catch (const json::type_error &) { + fmt::print(fg(fmt::color::red), "Key \"{}\" is of wrong type\n", key); + return false; + } +} + +/// @brief Get value from JSON object and store in out, setting success flag +/// @tparam T Type of output object +/// @param j JSON object +/// @param key Key to value +/// @param out Output object +/// @param success Success flag, set to false in case of failure +/// @return True if value was retrieved successfully, false otherwise +template bool get_to(const json &j, const std::string &key, T &out, bool &success) { + const bool ret = get_to(j, key, out); + if (!ret) { + success = false; + } + return ret; +} + +/// @brief Rebase path on base_dir +/// @param path Initial path (relative or absolute) +/// @param base_dir New base directory for relative path +/// @throw ConfigurationError: If path does not exist +void rebase_valid_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { + if (path.is_relative()) { + path = std::filesystem::weakly_canonical(base_dir / path); + } + + if (!std::filesystem::exists(path)) { + throw ConfigurationError{fmt::format("Path does not exist: {}", path.string())}; + } +} + +/// @brief Get a valid path from a JSON object +/// @param j JSON object +/// @param key Key to value +/// @param base_dir Base directory for relative path +/// @param out Output variable +/// @return True if value was retrieved successfully and is valid path, false otherwise +bool get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, + std::filesystem::path &out) { + if (!get_to(j, key, out)) { + return false; + } + + try { + rebase_valid_path(out, base_dir); + } catch (const ConfigurationError &) { + fmt::print(fg(fmt::color::red), "Could not find file {}", out.string()); + return false; + } + + return true; +} + +/// @brief Get a valid path from a JSON object +/// @param j JSON object +/// @param key Key to value +/// @param base_dir Base directory for relative path +/// @param out Output variable +/// @param success Success flag, set to false in case of failure +void get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, + std::filesystem::path &out, bool &success) { + if (!get_valid_path_to(j, key, base_dir, out)) { + success = false; + } +} + +/// @brief Load FileInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return FileInfo +/// @throw ConfigurationError: Invalid config file format +auto get_file_info(const json &j, const std::filesystem::path &base_dir) { + const auto dataset = get(j, "dataset"); + + bool success = true; + poco::FileInfo info; + get_valid_path_to(dataset, "name", base_dir, info.name, success); + get_to(dataset, "format", info.format, success); + get_to(dataset, "delimiter", info.delimiter, success); + get_to(dataset, "columns", info.columns, success); + if (!success) { + throw ConfigurationError{"Could not load input file info"}; + } + + return info; +} + +auto get_settings(const json &j) { + poco::SettingsInfo info; + if (!get_to(j, "settings", info)) { + throw ConfigurationError{"Could not load settings info"}; + } + + return info; +} + +/// @brief Load BaselineInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return BaselineInfo +/// @throw ConfigurationError: One or more files could not be found +auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { + const auto &adj = get(j, "baseline_adjustments"); + + bool success = true; + poco::BaselineInfo info; + get_to(adj, "format", info.format, success); + get_to(adj, "delimiter", info.delimiter, success); + get_to(adj, "encoding", info.encoding, success); + if (get_to(adj, "file_names", info.file_names, success)) { + // Rebase paths and check for errors + for (auto &[name, path] : info.file_names) { + try { + rebase_valid_path(path, base_dir); + fmt::print("{:<14}, file: {}\n", name, path.string()); + } catch (const ConfigurationError &) { + fmt::print(fg(fmt::color::red), "Could not find file: {}\n", path.string()); + success = false; + } + } + } + + if (!success) { + throw ConfigurationError{"Could not get baseline adjustments"}; + } + + return info; +} + +/// @brief Load interventions from running section +/// @param running Running section of JSON object +/// @param config Config object to update +/// @throw ConfigurationError: Could not load interventions +void load_interventions(const json &running, Configuration &config) { + const auto interventions = get(running, "interventions"); + + try { + // If the type of intervention is null, then there's nothing else to do + if (interventions.at("active_type_id").is_null()) { + return; + } + } catch (const std::out_of_range &) { + throw ConfigurationError{"Interventions section missing key \"active_type_id\""}; + } + + const auto active_type_id = [&interventions]() { + try { + return interventions["active_type_id"].get(); + } catch (const json::type_error &) { + throw ConfigurationError{"active_type_id key must be of type string"}; + } + }(); + + /* + * NB: This loads all of the policy scenario info from the JSON file, which is + * strictly speaking unnecessary, but it does mean that we can verify the data + * format is correct. + */ + std::unordered_map policy_types; + if (!get_to(interventions, "types", policy_types)) { + throw ConfigurationError{"Could not load policy types from interventions section"}; + } + + try { + config.intervention = policy_types.at(active_type_id); + config.intervention.identifier = active_type_id.to_string(); + config.has_active_intervention = true; + } catch (const std::out_of_range &) { + throw ConfigurationError{fmt::format("Unknown active intervention type identifier: {}", + active_type_id.to_string())}; + } +} + +void check_version(const json &j) { + int version; + if (!get_to(j, "version", version)) { + throw ConfigurationError{"File must have a schema version"}; + } + + if (version != 2) { + throw ConfigurationError{ + fmt::format("Configuration schema version: {} mismatch, supported: 2", version)}; + } +} + +void load_input_info(const json &j, Configuration &config, + const std::filesystem::path &config_dir) { + const auto inputs = get(j, "inputs"); + bool success = true; + + // Input dataset file + try { + config.file = get_file_info(inputs, config_dir); + fmt::print("Input dataset file: {}\n", config.file.name.string()); + } catch (const std::exception &e) { + success = false; + fmt::print(fg(fmt::color::red), "Could not load dataset file: {}\n", e.what()); + } + + // Settings + try { + config.settings = get_settings(inputs); + } catch (const std::exception &e) { + success = false; + fmt::print(fg(fmt::color::red), "Could not load settings info"); + } + + if (!success) { + throw ConfigurationError{"Could not load settings info"}; + } +} + +void load_modelling_info(const json &j, Configuration &config, + const std::filesystem::path &config_dir) { + bool success = true; + const auto modelling = get(j, "modelling"); + + auto &info = config.modelling; + get_to(modelling, "risk_factors", info.risk_factors, success); + + // Rebase paths and check for errors + if (get_to(modelling, "risk_factor_models", info.risk_factor_models, success)) { + for (auto &[type, path] : info.risk_factor_models) { + try { + rebase_valid_path(path, config_dir); + fmt::print("{:<14}, file: {}\n", type, path.string()); + } catch (const ConfigurationError &) { + success = false; + fmt::print(fg(fmt::color::red), "Adjustment type: {}, file: {} not found.\n", type, + path.string()); + } + } + } + + try { + info.baseline_adjustment = get_baseline_info(modelling, config_dir); + } catch (const std::exception &e) { + success = false; + fmt::print(fmt::fg(fmt::color::red), "Could not load baseline adjustment: {}\n", e.what()); + } + + try { + // SES mapping + // TODO: Maybe this needs its own helper function + config.ses = get(modelling, "ses_model").get(); + } catch (const std::exception &e) { + success = false; + fmt::print(fmt::fg(fmt::color::red), "Could not load SES mappings"); + } + + if (!success) { + throw ConfigurationError("Could not load modelling info"); + } +} + +void load_running_info(const json &j, Configuration &config) { + const auto running = get(j, "running"); + + bool success = true; + get_to(running, "start_time", config.start_time, success); + get_to(running, "stop_time", config.stop_time, success); + get_to(running, "trial_runs", config.trial_runs, success); + get_to(running, "sync_timeout_ms", config.sync_timeout_ms, success); + get_to(running, "diseases", config.diseases, success); + + { + // I copied this logic from the old code, but it seems strange to me. Why do we + // store multiple seeds but only use the first? -- Alex + std::vector seeds; + if (get_to(running, "seed", seeds, success) && !seeds.empty()) { + config.custom_seed = seeds[0]; + } + } + + // Intervention Policy + try { + load_interventions(running, config); + } catch (const ConfigurationError &e) { + success = false; + fmt::print(fmt::fg(fmt::color::red), "Could not load interventions: {}", e.what()); + } + + if (!success) { + throw ConfigurationError{"Could not load running info"}; + } +} + +void load_output_info(const json &j, Configuration &config) { + if (!get_to(j, "output", config.output)) { + throw ConfigurationError{"Could not load output info"}; + } + + config.output.folder = expand_environment_variables(config.output.folder); +} + +} // namespace host diff --git a/src/HealthGPS.Console/configuration_parsing.h b/src/HealthGPS.Console/configuration_parsing.h new file mode 100644 index 000000000..ff6a5984c --- /dev/null +++ b/src/HealthGPS.Console/configuration_parsing.h @@ -0,0 +1,37 @@ +#pragma once +#include "configuration.h" + +namespace host { +/// @brief Check the schema version and throw if invalid +/// @param j The root JSON object +/// @throw ConfigurationError: If version attribute is not present or invalid +void check_version(const nlohmann::json &j); + +/// @brief Load input dataset +/// @param j The root JSON object +/// @param config The config object to update +/// @param config_dir The directory of the config file +/// @throw ConfigurationError: Could not load input dataset +void load_input_info(const nlohmann::json &j, Configuration &config, + const std::filesystem::path &config_dir); + +/// @brief Load ModellingInfo from JSON +/// @param j The root JSON object +/// @param config The config object to update +/// @param config_dir The directory of the config file +/// @throw ConfigurationError: Could not load modelling info +void load_modelling_info(const nlohmann::json &j, Configuration &config, + const std::filesystem::path &config_dir); + +/// @brief Load running section of JSON object +/// @param j The root JSON object +/// @param config The config object to update +/// @throw ConfigurationError: Could not load running section +void load_running_info(const nlohmann::json &j, Configuration &config); + +/// @brief Load output section of JSON object +/// @param j The root JSON object +/// @param config The config object to update +/// @throw ConfigurationError: Could not load output info +void load_output_info(const nlohmann::json &j, Configuration &config); +} // namespace host \ No newline at end of file From 3d722905eb323bed0fcf4605a2b6c1ec48470157 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 13:44:39 +0100 Subject: [PATCH 15/46] Put configuration parsing helper function defs into own header This is so we can test them with GTest. --- src/HealthGPS.Console/CMakeLists.txt | 1 + .../configuration_parsing.cpp | 88 ++------------- .../configuration_parsing_helpers.h | 105 ++++++++++++++++++ 3 files changed, 116 insertions(+), 78 deletions(-) create mode 100644 src/HealthGPS.Console/configuration_parsing_helpers.h diff --git a/src/HealthGPS.Console/CMakeLists.txt b/src/HealthGPS.Console/CMakeLists.txt index ffe2027e8..787c424ff 100644 --- a/src/HealthGPS.Console/CMakeLists.txt +++ b/src/HealthGPS.Console/CMakeLists.txt @@ -19,6 +19,7 @@ target_sources(HealthGPS.Console "configuration.h" "configuration_parsing.cpp" "configuration_parsing.h" + "configuration_parsing_helpers.h" "csvparser.cpp" "csvparser.h" "event_monitor.cpp" diff --git a/src/HealthGPS.Console/configuration_parsing.cpp b/src/HealthGPS.Console/configuration_parsing.cpp index 1a36ff947..f3f024664 100644 --- a/src/HealthGPS.Console/configuration_parsing.cpp +++ b/src/HealthGPS.Console/configuration_parsing.cpp @@ -1,4 +1,5 @@ #include "configuration_parsing.h" +#include "configuration_parsing_helpers.h" #include "jsonparser.h" #include @@ -8,58 +9,15 @@ namespace host { using json = nlohmann::json; -/// @brief Load value from JSON, printing an error message if it fails -/// @param j JSON object -/// @param key Key to value -/// @throw ConfigurationError: Key not found -/// @return Key value -auto get(const json &j, const std::string &key) { +nlohmann::json get(const json &j, const std::string &key) { try { return j.at(key); - } catch (const std::out_of_range &) { + } catch (const std::exception &) { fmt::print(fg(fmt::color::red), "Missing key \"{}\"\n", key); throw ConfigurationError{fmt::format("Missing key \"{}\"", key)}; } } -/// @brief Get value from JSON object and store in out -/// @tparam T Type of output object -/// @param j JSON object -/// @param key Key to value -/// @param out Output object -/// @return True if value was retrieved successfully, false otherwise -template bool get_to(const json &j, const std::string &key, T &out) { - try { - j.at(key).get_to(out); - return true; - } catch (const std::out_of_range &) { - fmt::print(fg(fmt::color::red), "Missing key \"{}\"\n", key); - return false; - } catch (const json::type_error &) { - fmt::print(fg(fmt::color::red), "Key \"{}\" is of wrong type\n", key); - return false; - } -} - -/// @brief Get value from JSON object and store in out, setting success flag -/// @tparam T Type of output object -/// @param j JSON object -/// @param key Key to value -/// @param out Output object -/// @param success Success flag, set to false in case of failure -/// @return True if value was retrieved successfully, false otherwise -template bool get_to(const json &j, const std::string &key, T &out, bool &success) { - const bool ret = get_to(j, key, out); - if (!ret) { - success = false; - } - return ret; -} - -/// @brief Rebase path on base_dir -/// @param path Initial path (relative or absolute) -/// @param base_dir New base directory for relative path -/// @throw ConfigurationError: If path does not exist void rebase_valid_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { if (path.is_relative()) { path = std::filesystem::weakly_canonical(base_dir / path); @@ -70,12 +28,6 @@ void rebase_valid_path(std::filesystem::path &path, const std::filesystem::path } } -/// @brief Get a valid path from a JSON object -/// @param j JSON object -/// @param key Key to value -/// @param base_dir Base directory for relative path -/// @param out Output variable -/// @return True if value was retrieved successfully and is valid path, false otherwise bool get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, std::filesystem::path &out) { if (!get_to(j, key, out)) { @@ -92,12 +44,6 @@ bool get_valid_path_to(const json &j, const std::string &key, const std::filesys return true; } -/// @brief Get a valid path from a JSON object -/// @param j JSON object -/// @param key Key to value -/// @param base_dir Base directory for relative path -/// @param out Output variable -/// @param success Success flag, set to false in case of failure void get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, std::filesystem::path &out, bool &success) { if (!get_valid_path_to(j, key, base_dir, out)) { @@ -105,12 +51,7 @@ void get_valid_path_to(const json &j, const std::string &key, const std::filesys } } -/// @brief Load FileInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return FileInfo -/// @throw ConfigurationError: Invalid config file format -auto get_file_info(const json &j, const std::filesystem::path &base_dir) { +poco::FileInfo get_file_info(const json &j, const std::filesystem::path &base_dir) { const auto dataset = get(j, "dataset"); bool success = true; @@ -126,7 +67,7 @@ auto get_file_info(const json &j, const std::filesystem::path &base_dir) { return info; } -auto get_settings(const json &j) { +poco::SettingsInfo get_settings(const json &j) { poco::SettingsInfo info; if (!get_to(j, "settings", info)) { throw ConfigurationError{"Could not load settings info"}; @@ -135,12 +76,7 @@ auto get_settings(const json &j) { return info; } -/// @brief Load BaselineInfo from JSON -/// @param j Input JSON -/// @param base_dir Base folder -/// @return BaselineInfo -/// @throw ConfigurationError: One or more files could not be found -auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { +poco::BaselineInfo get_baseline_info(const json &j, const std::filesystem::path &base_dir) { const auto &adj = get(j, "baseline_adjustments"); bool success = true; @@ -168,10 +104,6 @@ auto get_baseline_info(const json &j, const std::filesystem::path &base_dir) { return info; } -/// @brief Load interventions from running section -/// @param running Running section of JSON object -/// @param config Config object to update -/// @throw ConfigurationError: Could not load interventions void load_interventions(const json &running, Configuration &config) { const auto interventions = get(running, "interventions"); @@ -180,7 +112,7 @@ void load_interventions(const json &running, Configuration &config) { if (interventions.at("active_type_id").is_null()) { return; } - } catch (const std::out_of_range &) { + } catch (const json::out_of_range &) { throw ConfigurationError{"Interventions section missing key \"active_type_id\""}; } @@ -206,7 +138,7 @@ void load_interventions(const json &running, Configuration &config) { config.intervention = policy_types.at(active_type_id); config.intervention.identifier = active_type_id.to_string(); config.has_active_intervention = true; - } catch (const std::out_of_range &) { + } catch (const json::out_of_range &) { throw ConfigurationError{fmt::format("Unknown active intervention type identifier: {}", active_type_id.to_string())}; } @@ -241,7 +173,7 @@ void load_input_info(const json &j, Configuration &config, // Settings try { config.settings = get_settings(inputs); - } catch (const std::exception &e) { + } catch (const std::exception &) { success = false; fmt::print(fg(fmt::color::red), "Could not load settings info"); } @@ -284,7 +216,7 @@ void load_modelling_info(const json &j, Configuration &config, // SES mapping // TODO: Maybe this needs its own helper function config.ses = get(modelling, "ses_model").get(); - } catch (const std::exception &e) { + } catch (const std::exception &) { success = false; fmt::print(fmt::fg(fmt::color::red), "Could not load SES mappings"); } diff --git a/src/HealthGPS.Console/configuration_parsing_helpers.h b/src/HealthGPS.Console/configuration_parsing_helpers.h new file mode 100644 index 000000000..b11cbe9e8 --- /dev/null +++ b/src/HealthGPS.Console/configuration_parsing_helpers.h @@ -0,0 +1,105 @@ +#pragma once +#include "configuration.h" +#include "poco.h" + +#include +#include + +#include +#include + +namespace host { +/// @brief Load value from JSON, printing an error message if it fails +/// @param j JSON object +/// @param key Key to value +/// @throw ConfigurationError: Key not found +/// @return Key value +nlohmann::json get(const nlohmann::json &j, const std::string &key); + +/// @brief Get value from JSON object and store in out +/// @tparam T Type of output object +/// @param j JSON object +/// @param key Key to value +/// @param out Output object +/// @return True if value was retrieved successfully, false otherwise +template bool get_to(const nlohmann::json &j, const std::string &key, T &out) { + try { + out = j.at(key).get(); + return true; + } catch (const nlohmann::json::out_of_range &) { + fmt::print(fg(fmt::color::red), "Missing key \"{}\"\n", key); + return false; + } catch (const nlohmann::json::type_error &) { + fmt::print(fg(fmt::color::red), "Key \"{}\" is of wrong type\n", key); + return false; + } +} + +/// @brief Get value from JSON object and store in out, setting success flag +/// @tparam T Type of output object +/// @param j JSON object +/// @param key Key to value +/// @param out Output object +/// @param success Success flag, set to false in case of failure +/// @return True if value was retrieved successfully, false otherwise +template +bool get_to(const nlohmann::json &j, const std::string &key, T &out, bool &success) { + const bool ret = get_to(j, key, out); + if (!ret) { + success = false; + } + return ret; +} + +/// @brief Rebase path on base_dir +/// @param path Initial path (relative or absolute) +/// @param base_dir New base directory for relative path +/// @throw ConfigurationError: If path does not exist +void rebase_valid_path(std::filesystem::path &path, const std::filesystem::path &base_dir); + +/// @brief Get a valid path from a JSON object +/// @param j JSON object +/// @param key Key to value +/// @param base_dir Base directory for relative path +/// @param out Output variable +/// @return True if value was retrieved successfully and is valid path, false otherwise +bool get_valid_path_to(const nlohmann::json &j, const std::string &key, + const std::filesystem::path &base_dir, std::filesystem::path &out); + +/// @brief Get a valid path from a JSON object +/// @param j JSON object +/// @param key Key to value +/// @param base_dir Base directory for relative path +/// @param out Output variable +/// @param success Success flag, set to false in case of failure +void get_valid_path_to(const nlohmann::json &j, const std::string &key, + const std::filesystem::path &base_dir, std::filesystem::path &out, + bool &success); + +/// @brief Load FileInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return FileInfo +/// @throw ConfigurationError: Invalid config file format +poco::FileInfo get_file_info(const nlohmann::json &j, const std::filesystem::path &base_dir); + +/// @brief Load settings section of JSON +/// @param j Input JSON +/// @return SettingsInfo +/// @throw ConfigurationError: Could not load settings +poco::SettingsInfo get_settings(const nlohmann::json &j); + +/// @brief Load BaselineInfo from JSON +/// @param j Input JSON +/// @param base_dir Base folder +/// @return BaselineInfo +/// @throw ConfigurationError: One or more files could not be found +poco::BaselineInfo get_baseline_info(const nlohmann::json &j, + const std::filesystem::path &base_dir); + +/// @brief Load interventions from running section +/// @param running Running section of JSON object +/// @param config Config object to update +/// @throw ConfigurationError: Could not load interventions +void load_interventions(const nlohmann::json &running, Configuration &config); +} // namespace host From edc505d360b73ffcb5dba82ea6bfa24862761c95 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 17:43:38 +0100 Subject: [PATCH 16/46] Rename some functions and mark noexcept --- .../configuration_parsing.cpp | 24 +++++++++++-------- .../configuration_parsing_helpers.h | 19 ++++++++------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/HealthGPS.Console/configuration_parsing.cpp b/src/HealthGPS.Console/configuration_parsing.cpp index f3f024664..1631149db 100644 --- a/src/HealthGPS.Console/configuration_parsing.cpp +++ b/src/HealthGPS.Console/configuration_parsing.cpp @@ -20,7 +20,11 @@ nlohmann::json get(const json &j, const std::string &key) { void rebase_valid_path(std::filesystem::path &path, const std::filesystem::path &base_dir) { if (path.is_relative()) { - path = std::filesystem::weakly_canonical(base_dir / path); + try { + path = std::filesystem::weakly_canonical(base_dir / path); + } catch (const std::filesystem::filesystem_error &e) { + throw ConfigurationError{fmt::format("OS error while reading {}", path.string())}; + } } if (!std::filesystem::exists(path)) { @@ -28,25 +32,25 @@ void rebase_valid_path(std::filesystem::path &path, const std::filesystem::path } } -bool get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, - std::filesystem::path &out) { - if (!get_to(j, key, out)) { +bool rebase_valid_path_to(const json &j, const std::string &key, std::filesystem::path &path, + const std::filesystem::path &base_dir) noexcept { + if (!get_to(j, key, path)) { return false; } try { - rebase_valid_path(out, base_dir); + rebase_valid_path(path, base_dir); } catch (const ConfigurationError &) { - fmt::print(fg(fmt::color::red), "Could not find file {}", out.string()); + fmt::print(fg(fmt::color::red), "Could not find file {}\n", path.string()); return false; } return true; } -void get_valid_path_to(const json &j, const std::string &key, const std::filesystem::path &base_dir, - std::filesystem::path &out, bool &success) { - if (!get_valid_path_to(j, key, base_dir, out)) { +void rebase_valid_path_to(const json &j, const std::string &key, std::filesystem::path &path, + const std::filesystem::path &base_dir, bool &success) noexcept { + if (!rebase_valid_path_to(j, key, path, base_dir)) { success = false; } } @@ -56,7 +60,7 @@ poco::FileInfo get_file_info(const json &j, const std::filesystem::path &base_di bool success = true; poco::FileInfo info; - get_valid_path_to(dataset, "name", base_dir, info.name, success); + rebase_valid_path_to(dataset, "name", info.name, base_dir, success); get_to(dataset, "format", info.format, success); get_to(dataset, "delimiter", info.delimiter, success); get_to(dataset, "columns", info.columns, success); diff --git a/src/HealthGPS.Console/configuration_parsing_helpers.h b/src/HealthGPS.Console/configuration_parsing_helpers.h index b11cbe9e8..438ff1031 100644 --- a/src/HealthGPS.Console/configuration_parsing_helpers.h +++ b/src/HealthGPS.Console/configuration_parsing_helpers.h @@ -22,7 +22,7 @@ nlohmann::json get(const nlohmann::json &j, const std::string &key); /// @param key Key to value /// @param out Output object /// @return True if value was retrieved successfully, false otherwise -template bool get_to(const nlohmann::json &j, const std::string &key, T &out) { +template bool get_to(const nlohmann::json &j, const std::string &key, T &out) noexcept { try { out = j.at(key).get(); return true; @@ -43,7 +43,7 @@ template bool get_to(const nlohmann::json &j, const std::string &key, /// @param success Success flag, set to false in case of failure /// @return True if value was retrieved successfully, false otherwise template -bool get_to(const nlohmann::json &j, const std::string &key, T &out, bool &success) { +bool get_to(const nlohmann::json &j, const std::string &key, T &out, bool &success) noexcept { const bool ret = get_to(j, key, out); if (!ret) { success = false; @@ -61,20 +61,21 @@ void rebase_valid_path(std::filesystem::path &path, const std::filesystem::path /// @param j JSON object /// @param key Key to value /// @param base_dir Base directory for relative path -/// @param out Output variable +/// @param path Output variable /// @return True if value was retrieved successfully and is valid path, false otherwise -bool get_valid_path_to(const nlohmann::json &j, const std::string &key, - const std::filesystem::path &base_dir, std::filesystem::path &out); +bool rebase_valid_path_to(const nlohmann::json &j, const std::string &key, + std::filesystem::path &path, + const std::filesystem::path &base_dir) noexcept; /// @brief Get a valid path from a JSON object /// @param j JSON object /// @param key Key to value /// @param base_dir Base directory for relative path -/// @param out Output variable +/// @param path Output variable /// @param success Success flag, set to false in case of failure -void get_valid_path_to(const nlohmann::json &j, const std::string &key, - const std::filesystem::path &base_dir, std::filesystem::path &out, - bool &success); +void rebase_valid_path_to(const nlohmann::json &j, const std::string &key, + std::filesystem::path &out, const std::filesystem::path &base_dir, + bool &success) noexcept; /// @brief Load FileInfo from JSON /// @param j Input JSON From 04eab1eebc0cc09130928cfcb22361f630b88acc Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 14:03:06 +0100 Subject: [PATCH 17/46] Remove unused API version functionality --- src/HealthGPS.Core/CMakeLists.txt | 2 -- src/HealthGPS.Core/api.h | 1 - src/HealthGPS.Core/version.cpp | 52 ------------------------------ src/HealthGPS.Core/version.h | 53 ------------------------------- src/HealthGPS.Tests/Core.Test.cpp | 20 ------------ 5 files changed, 128 deletions(-) delete mode 100644 src/HealthGPS.Core/version.cpp delete mode 100644 src/HealthGPS.Core/version.h diff --git a/src/HealthGPS.Core/CMakeLists.txt b/src/HealthGPS.Core/CMakeLists.txt index 9c78f4c8b..f0dbc0ffa 100644 --- a/src/HealthGPS.Core/CMakeLists.txt +++ b/src/HealthGPS.Core/CMakeLists.txt @@ -15,8 +15,6 @@ target_sources(HealthGPS.Core "univariate_summary.h" "math_util.cpp" "math_util.h" - "version.cpp" - "version.h" "identifier.cpp" "identifier.h" diff --git a/src/HealthGPS.Core/api.h b/src/HealthGPS.Core/api.h index e0178fe9a..5d5ff4942 100644 --- a/src/HealthGPS.Core/api.h +++ b/src/HealthGPS.Core/api.h @@ -10,7 +10,6 @@ #include "interval.h" #include "poco.h" #include "univariate_summary.h" -#include "version.h" #include "visitor.h" namespace hgps { diff --git a/src/HealthGPS.Core/version.cpp b/src/HealthGPS.Core/version.cpp deleted file mode 100644 index c54bb96b3..000000000 --- a/src/HealthGPS.Core/version.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "version.h" -#include -#include - -namespace hgps::core { - -int Version::GetMajor() { return int(API_MAJOR); } - -int Version::GetMinor() { return int(API_MINOR); } - -int Version::GetPatch() { return int(API_PATCH); } - -std::string Version::GetVersion() { - static std::string version(""); - - if (version.empty()) { - // Cache the version string - std::ostringstream stream; - stream << API_MAJOR << "." << API_MINOR << "." << API_PATCH; - - version = stream.str(); - } - - return version; -} - -bool Version::IsAtLeast(int major, int minor, int patch) { - if (API_MAJOR < major) - return false; - if (API_MAJOR > major) - return true; - if (API_MINOR < minor) - return false; - if (API_MINOR > minor) - return true; - if (API_PATCH < patch) - return false; - return true; -} - -bool Version::HasFeature(const std::string &name) { - static std::set features; - - if (features.empty()) { - // Cache the feature list - features.insert("COUNTRY"); - features.insert("THREADSAFE"); - } - - return features.find(name) != features.end(); -} -} // namespace hgps::core diff --git a/src/HealthGPS.Core/version.h b/src/HealthGPS.Core/version.h deleted file mode 100644 index da026b6fd..000000000 --- a/src/HealthGPS.Core/version.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include - -namespace hgps::core { - -/// @brief Version major number -constexpr auto API_MAJOR = 1; - -/// @brief Version minor number -constexpr auto API_MINOR = 2; - -/// @brief Version patch number -constexpr auto API_PATCH = 0; - -/// @brief Application Programming Interface (API) version data type -class Version { - public: - Version() = delete; - Version(const Version &) = delete; - Version &operator=(const Version &) = delete; - Version(const Version &&) = delete; - Version &operator=(const Version &&) = delete; - - /// @brief Gets the API major version - /// @return The major version value - static int GetMajor(); - - /// @brief Gets the API minor version - /// @return The minor version value - static int GetMinor(); - - /// @brief Gets the API patch version - /// @return The patch version value - static int GetPatch(); - - /// @brief Creates a string representation of API version - /// @return The version string - static std::string GetVersion(); - - /// @brief Validates the API version compatibility - /// @param major Minimum required major version - /// @param minor Minimum required minor version - /// @param patch Minimum required patch version - /// @return true if the versions are compatible, otherwise false - static bool IsAtLeast(int major, int minor, int patch); - - /// @brief Checks whether the API version has specific features - /// @param name The required feature name - /// @return true if the feature is available, otherwise false - static bool HasFeature(const std::string &name); -}; -} // namespace hgps::core diff --git a/src/HealthGPS.Tests/Core.Test.cpp b/src/HealthGPS.Tests/Core.Test.cpp index 5152cf4fa..38fa791a4 100644 --- a/src/HealthGPS.Tests/Core.Test.cpp +++ b/src/HealthGPS.Tests/Core.Test.cpp @@ -5,26 +5,6 @@ #include #include -TEST(TestCore, CurrentApiVersion) { - using namespace hgps::core; - - std::stringstream ss; - ss << API_MAJOR << "." << API_MINOR << "." << API_PATCH; - - EXPECT_EQ(API_MAJOR, Version::GetMajor()); - EXPECT_EQ(API_MINOR, Version::GetMinor()); - EXPECT_EQ(API_PATCH, Version::GetPatch()); - EXPECT_EQ(ss.str(), Version::GetVersion()); - - EXPECT_TRUE(Version::HasFeature("COUNTRY")); - EXPECT_FALSE(Version::HasFeature("DISEASES")); - - EXPECT_TRUE(Version::IsAtLeast(API_MAJOR, API_MINOR, API_PATCH)); - EXPECT_FALSE(Version::IsAtLeast(API_MAJOR + 1, 0, 0)); - EXPECT_FALSE(Version::IsAtLeast(API_MAJOR, API_MINOR + 1, API_PATCH)); - EXPECT_FALSE(Version::IsAtLeast(API_MAJOR, API_MINOR, API_PATCH + 1)); -} - TEST(TestCore, CreateCountry) { using namespace hgps::core; From a7ee4b5ebba1f1eff7e0e4f1580b424fce98e293 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 14:05:24 +0100 Subject: [PATCH 18/46] Move version.h.in to src/HealthGPS It's common functionality, not specific to the console app. --- src/HealthGPS.Console/CMakeLists.txt | 8 -------- src/HealthGPS/CMakeLists.txt | 11 +++++++++++ src/{HealthGPS.Console => HealthGPS}/version.h.in | 0 src/{HealthGPS.Console => HealthGPS}/versioninfo.rc | 0 4 files changed, 11 insertions(+), 8 deletions(-) rename src/{HealthGPS.Console => HealthGPS}/version.h.in (100%) rename src/{HealthGPS.Console => HealthGPS}/versioninfo.rc (100%) diff --git a/src/HealthGPS.Console/CMakeLists.txt b/src/HealthGPS.Console/CMakeLists.txt index 787c424ff..297302dc0 100644 --- a/src/HealthGPS.Console/CMakeLists.txt +++ b/src/HealthGPS.Console/CMakeLists.txt @@ -8,7 +8,6 @@ find_package(Threads REQUIRED) add_executable(HealthGPS.Console "") target_compile_features(HealthGPS.Console PUBLIC cxx_std_${CMAKE_CXX_STANDARD}) -configure_file("version.h.in" "version.h" ESCAPE_QUOTES) include_directories(${CMAKE_CURRENT_BINARY_DIR}) target_sources(HealthGPS.Console @@ -37,13 +36,6 @@ target_sources(HealthGPS.Console "riskmodel.h" "poco.h") -# Under Windows, we also include a resource file to the build -if(WIN32) - # Make sure that the resource file is seen as an RC file to be compiled with a resource compiler, not a C++ compiler - set_source_files_properties(versioninfo.rc LANGUAGE RC) - target_sources(HealthGPS.Console PRIVATE versioninfo.rc) -endif(WIN32) - target_link_libraries(HealthGPS.Console PRIVATE HealthGPS.Core diff --git a/src/HealthGPS/CMakeLists.txt b/src/HealthGPS/CMakeLists.txt index f19bb1057..b49534fd7 100644 --- a/src/HealthGPS/CMakeLists.txt +++ b/src/HealthGPS/CMakeLists.txt @@ -4,6 +4,8 @@ find_package(crossguid CONFIG REQUIRED) add_library(HealthGPS STATIC "") target_compile_features(HealthGPS PUBLIC cxx_std_${CMAKE_CXX_STANDARD}) +configure_file("version.h.in" "version.h" ESCAPE_QUOTES) + target_sources(HealthGPS PRIVATE "analysis_module.cpp" @@ -124,14 +126,23 @@ target_sources(HealthGPS "simulation.h" "sync_message.h" "two_step_value.h" + "version.h" "weight_category.h" "weight_model.cpp" "weight_model.h") +# Under Windows, we also include a resource file to the build +if(WIN32) + # Make sure that the resource file is seen as an RC file to be compiled with a resource compiler, not a C++ compiler + set_source_files_properties(versioninfo.rc LANGUAGE RC) + target_sources(HealthGPS PRIVATE versioninfo.rc) +endif(WIN32) + target_link_libraries(HealthGPS PRIVATE HealthGPS.Core fmt::fmt crossguid) +target_include_directories(HealthGPS PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) set(ROOT_NAMESPACE hgps) diff --git a/src/HealthGPS.Console/version.h.in b/src/HealthGPS/version.h.in similarity index 100% rename from src/HealthGPS.Console/version.h.in rename to src/HealthGPS/version.h.in diff --git a/src/HealthGPS.Console/versioninfo.rc b/src/HealthGPS/versioninfo.rc similarity index 100% rename from src/HealthGPS.Console/versioninfo.rc rename to src/HealthGPS/versioninfo.rc From 7ff7cc19bc32bd9a39e2c7c8d1bdf394880f1433 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Wed, 9 Aug 2023 14:22:14 +0100 Subject: [PATCH 19/46] Build most console functionality as lib for testability --- src/HealthGPS.Console/CMakeLists.txt | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/HealthGPS.Console/CMakeLists.txt b/src/HealthGPS.Console/CMakeLists.txt index 297302dc0..50db7f52f 100644 --- a/src/HealthGPS.Console/CMakeLists.txt +++ b/src/HealthGPS.Console/CMakeLists.txt @@ -5,13 +5,8 @@ find_package(TBB CONFIG REQUIRED) find_path(RAPIDCSV_INCLUDE_DIRS "rapidcsv.h") find_package(Threads REQUIRED) -add_executable(HealthGPS.Console "") -target_compile_features(HealthGPS.Console PUBLIC cxx_std_${CMAKE_CXX_STANDARD}) - -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -target_sources(HealthGPS.Console - PRIVATE +add_executable(HealthGPS.Console "program.cpp") +add_library(HealthGPS.LibConsole STATIC "command_options.cpp" "command_options.h" "configuration.cpp" @@ -36,7 +31,21 @@ target_sources(HealthGPS.Console "riskmodel.h" "poco.h") +target_include_directories(HealthGPS.LibConsole PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) + target_link_libraries(HealthGPS.Console +PRIVATE + HealthGPS.Core + HealthGPS.Datastore + HealthGPS.LibConsole + HealthGPS + fmt::fmt + Threads::Threads + cxxopts::cxxopts + nlohmann_json::nlohmann_json + TBB::tbb) + +target_link_libraries(HealthGPS.LibConsole PRIVATE HealthGPS.Core HealthGPS.Datastore @@ -52,4 +61,4 @@ if(WIN32) install(IMPORTED_RUNTIME_ARTIFACTS fmt::fmt) endif() -set(ROOT_NAMESPACE hgps) \ No newline at end of file +set(ROOT_NAMESPACE hgps) From c6306a1116b6c11b703cbb73e492b1a5b7bb1cc8 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Fri, 11 Aug 2023 11:18:49 +0100 Subject: [PATCH 20/46] Spelling --- src/HealthGPS.Tests/Datastore.Test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/HealthGPS.Tests/Datastore.Test.cpp b/src/HealthGPS.Tests/Datastore.Test.cpp index 55965e804..f26173447 100644 --- a/src/HealthGPS.Tests/Datastore.Test.cpp +++ b/src/HealthGPS.Tests/Datastore.Test.cpp @@ -143,7 +143,7 @@ TEST_F(DatastoreTest, GetDiseaseInfoMissingThrowsException) { EXPECT_THROW(manager.get_disease_info("FAKE"), std::invalid_argument); } -TEST_F(DatastoreTest, RetrieveDeseasesTypeInInfo) { +TEST_F(DatastoreTest, RetrieveDiseasesTypeInInfo) { using namespace hgps::core; auto diseases = manager.get_diseases(); @@ -158,7 +158,7 @@ TEST_F(DatastoreTest, RetrieveDeseasesTypeInInfo) { ASSERT_GT(cancer_count, 0); } -TEST_F(DatastoreTest, RetrieveDeseaseDefinition) { +TEST_F(DatastoreTest, RetrieveDiseaseDefinition) { auto diseases = manager.get_diseases(); for (auto &item : diseases) { auto entity = manager.get_disease(item, uk); @@ -171,7 +171,7 @@ TEST_F(DatastoreTest, RetrieveDeseaseDefinition) { } } -TEST_F(DatastoreTest, RetrieveDeseaseDefinitionIsEmpty) { +TEST_F(DatastoreTest, RetrieveDiseaseDefinitionIsEmpty) { using namespace hgps::core; auto info = DiseaseInfo{.group = DiseaseGroup::other, From 3934a7fd363fa16846d90d0f5c20ff12ddea6b8e Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Fri, 11 Aug 2023 12:35:46 +0100 Subject: [PATCH 21/46] Add JSON (de)serialisation code for Interval<> class --- src/HealthGPS.Console/jsonparser.h | 37 ++++++++++++ src/HealthGPS.Tests/CMakeLists.txt | 1 + src/HealthGPS.Tests/JsonParser.Test.cpp | 79 +++++++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 src/HealthGPS.Tests/JsonParser.Test.cpp diff --git a/src/HealthGPS.Console/jsonparser.h b/src/HealthGPS.Console/jsonparser.h index 9139db436..f026ad893 100644 --- a/src/HealthGPS.Console/jsonparser.h +++ b/src/HealthGPS.Console/jsonparser.h @@ -73,6 +73,23 @@ void from_json(const json &j, OutputInfo &p); } // namespace host::poco +namespace hgps::core { +using json = nlohmann::json; + +template void to_json(json &j, const Interval &interval) { + j = json::array({interval.lower(), interval.upper()}); +} + +template void from_json(const json &j, Interval &interval) { + const auto vec = j.get>(); + if (vec.size() != 2) { + throw json::type_error::create(302, "Interval arrays must have only two elements", nullptr); + } + + interval = Interval{vec[0], vec[1]}; +} +} // namespace hgps::core + namespace std { // Optional parameters @@ -91,4 +108,24 @@ template void from_json(const nlohmann::json &j, std::optional & } } +template +void to_json(nlohmann::json &j, const std::optional> &p) { + if (p) { + j = *p; + } else { + // Null interval expressed as empty JSON array + j = nlohmann::json::array(); + } +} + +template +void from_json(const nlohmann::json &j, std::optional> &p) { + // Treat null JSON values and empty arrays as a null Interval + if (j.is_null() || (j.is_array() && j.empty())) { + p = std::nullopt; + } else { + p = j.get>(); + } +} + } // namespace std diff --git a/src/HealthGPS.Tests/CMakeLists.txt b/src/HealthGPS.Tests/CMakeLists.txt index 19adaaa3f..decee4656 100644 --- a/src/HealthGPS.Tests/CMakeLists.txt +++ b/src/HealthGPS.Tests/CMakeLists.txt @@ -28,6 +28,7 @@ target_sources(HealthGPS.Tests "EventAggregator.Test.cpp" "HealthGPS.Test.cpp" "HierarchicalMapping.Test.cpp" + "JsonParser.Test.cpp" "Identifier.Test.cpp" "LifeTable.Test.cpp" "Map2d.Test.cpp" diff --git a/src/HealthGPS.Tests/JsonParser.Test.cpp b/src/HealthGPS.Tests/JsonParser.Test.cpp new file mode 100644 index 000000000..5ab95c7f9 --- /dev/null +++ b/src/HealthGPS.Tests/JsonParser.Test.cpp @@ -0,0 +1,79 @@ +#include "HealthGPS.Console/jsonparser.h" + +#include "pch.h" + +using namespace hgps::core; +using json = nlohmann::json; + +TEST(JsonParser, IntervalFromJson) { + { + // Successful parsing of regular interval + IntegerInterval interval; + const auto j = json::array({0, 1}); + EXPECT_NO_THROW(j.get_to(interval)); + EXPECT_EQ(interval, IntegerInterval(0, 1)); + } + + { + // Fail to parse empty array + IntegerInterval interval; + const auto j = json::array(); + EXPECT_THROW(j.get_to(interval), json::type_error); + } + + { + // Fail to parse array with > 2 elements + IntegerInterval interval; + const auto j = json::array({0, 1, 2}); + EXPECT_THROW(j.get_to(interval), json::type_error); + } +} + +TEST(JsonParser, IntervalToJson) { + // Convert to JSON and back again + const IntegerInterval interval{0, 1}; + const json j = interval; + EXPECT_EQ(j.get(), interval); +} + +TEST(JsonParser, OptionalIntervalFromJson) { + { + // Successful parsing of non-null interval + std::optional interval; + const auto j = json::array({0, 1}); + EXPECT_NO_THROW(j.get_to(interval)); + EXPECT_EQ(interval, IntegerInterval(0, 1)); + } + + { + // Successful parsing of null JSON value + std::optional interval; + const auto j = json{}; + EXPECT_NO_THROW(j.get_to(interval)); + EXPECT_FALSE(interval.has_value()); + } + + { + // Successful parsing of empty JSON array (equivalent to null) + std::optional interval; + const auto j = json::array(); + EXPECT_NO_THROW(j.get_to(interval)); + EXPECT_FALSE(interval.has_value()); + } +} + +TEST(JsonParser, OptionalIntervalToJson) { + { + // Null interval + const std::optional interval; + const json j = interval; + EXPECT_EQ(j.get(), interval); + } + + { + // Non-null interval + const auto interval = std::make_optional(0, 1); + const json j = interval; + EXPECT_EQ(j.get(), interval); + } +} \ No newline at end of file From 3111772ee4910a66e3e5c7e517ec4a131badede6 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Fri, 11 Aug 2023 12:36:18 +0100 Subject: [PATCH 22/46] Make SettingsInfo.age_range an IntegerInterval --- src/HealthGPS.Console/configuration.cpp | 5 +---- src/HealthGPS.Console/model_parser.cpp | 8 +++----- src/HealthGPS.Console/poco.h | 6 ++++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index b918e57c3..c02c440ae 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -131,9 +131,6 @@ std::vector get_diseases_info(core::Datastore &data_api, Conf ModelInput create_model_input(core::DataTable &input_table, core::Country country, Configuration &config, std::vector diseases) { // Create simulation configuration - auto age_range = - core::IntegerInterval(config.settings.age_range.front(), config.settings.age_range.back()); - auto comorbidities = config.output.comorbidities; auto diseases_number = static_cast(diseases.size()); if (comorbidities > diseases_number) { @@ -142,7 +139,7 @@ ModelInput create_model_input(core::DataTable &input_table, core::Country countr config.output.comorbidities, comorbidities); } - auto settings = Settings(country, config.settings.size_fraction, age_range); + auto settings = Settings(country, config.settings.size_fraction, config.settings.age_range); auto job_custom_seed = create_job_seed(config.job_id, config.custom_seed); auto run_info = RunInfo{ .start_time = config.start_time, diff --git a/src/HealthGPS.Console/model_parser.cpp b/src/HealthGPS.Console/model_parser.cpp index 490a6abbf..4ddaf9f7d 100644 --- a/src/HealthGPS.Console/model_parser.cpp +++ b/src/HealthGPS.Console/model_parser.cpp @@ -238,7 +238,7 @@ load_newebm_risk_model_definition(const poco::json &opt, const poco::SettingsInf } // Load M/F average heights for age. - unsigned int max_age = settings.age_range.back(); + const auto max_age = static_cast(settings.age_range.upper()); auto male_height = opt["AgeMeanHeight"]["Male"].get>(); auto female_height = opt["AgeMeanHeight"]["Female"].get>(); if (male_height.size() <= max_age) { @@ -301,15 +301,13 @@ void register_risk_factor_model_definitions(hgps::CachedRepository &repository, } auto adjustment = load_baseline_adjustments(info.baseline_adjustment); - auto age_range = - hgps::core::IntegerInterval(settings.age_range.front(), settings.age_range.back()); - auto max_age = static_cast(age_range.upper()); + auto max_age = static_cast(settings.age_range.upper()); for (const auto &table : adjustment.values) { for (const auto &item : table.second) { if (item.second.size() <= max_age) { fmt::print(fg(fmt::color::red), "Baseline adjustment files data must cover age range: [{}].\n", - age_range.to_string()); + settings.age_range.to_string()); throw std::invalid_argument( "Baseline adjustment file must cover the required age range."); } diff --git a/src/HealthGPS.Console/poco.h b/src/HealthGPS.Console/poco.h index a7a29b1d2..c3f21bebe 100644 --- a/src/HealthGPS.Console/poco.h +++ b/src/HealthGPS.Console/poco.h @@ -1,4 +1,6 @@ #pragma once +#include "HealthGPS.Core/interval.h" + #include #include #include @@ -15,9 +17,9 @@ struct FileInfo { }; struct SettingsInfo { - std::string country{}; + std::string country; + hgps::core::IntegerInterval age_range; float size_fraction{}; - std::vector age_range; }; struct SESInfo { From 35b93a2d760ba9b24f531a02bb205c45bd92eb86 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Fri, 11 Aug 2023 15:17:58 +0100 Subject: [PATCH 23/46] Base OptionalRange on Interval class and rename --- src/HealthGPS/mapping.cpp | 6 +++--- src/HealthGPS/mapping.h | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/HealthGPS/mapping.cpp b/src/HealthGPS/mapping.cpp index 171404782..66403fbe2 100644 --- a/src/HealthGPS/mapping.cpp +++ b/src/HealthGPS/mapping.cpp @@ -8,7 +8,7 @@ namespace hgps { -MappingEntry::MappingEntry(std::string name, int level, OptionalRange range) +MappingEntry::MappingEntry(std::string name, int level, OptionalInterval range) : name_{std::move(name)}, name_key_{name_}, level_{level}, range_{std::move(range)} {} const std::string &MappingEntry::name() const noexcept { return name_; } @@ -17,11 +17,11 @@ int MappingEntry::level() const noexcept { return level_; } const core::Identifier &MappingEntry::key() const noexcept { return name_key_; } -const OptionalRange &MappingEntry::range() const noexcept { return range_; } +const OptionalInterval &MappingEntry::range() const noexcept { return range_; } double MappingEntry::get_bounded_value(const double &value) const noexcept { if (range_.has_value()) { - return std::min(std::max(value, range_->first), range_->second); + return std::clamp(value, range_->lower(), range_->upper()); } return value; } diff --git a/src/HealthGPS/mapping.h b/src/HealthGPS/mapping.h index 8ee51ff76..37fb51249 100644 --- a/src/HealthGPS/mapping.h +++ b/src/HealthGPS/mapping.h @@ -1,5 +1,6 @@ #pragma once #include "HealthGPS.Core/identifier.h" +#include "HealthGPS.Core/interval.h" #include #include @@ -12,7 +13,7 @@ namespace hgps { inline const core::Identifier InterceptKey = core::Identifier{"intercept"}; /// @brief Optional Range of doubles data type -using OptionalRange = std::optional>; +using OptionalInterval = std::optional; /// @brief Defines risk factor mapping entry data type /// @@ -26,7 +27,7 @@ class MappingEntry { /// @param name Risk factor name /// @param level The hierarchical level /// @param range The factor range - MappingEntry(std::string name, int level, OptionalRange range = {}); + MappingEntry(std::string name, int level, OptionalInterval range = {}); /// @brief Gets the factor name /// @return Factor name @@ -42,7 +43,7 @@ class MappingEntry { /// @brief Gets the factor allowed values range /// @return Factor values range - const OptionalRange &range() const noexcept; + const OptionalInterval &range() const noexcept; /// @brief Adjusts a value to the factor range, if provided /// @param value The value to adjust @@ -53,7 +54,7 @@ class MappingEntry { std::string name_; core::Identifier name_key_; int level_{}; - OptionalRange range_; + OptionalInterval range_; }; /// @brief Defines the hierarchical model mapping data type From 9557033199437ab818697d6b83a33fb8445a7197 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Fri, 11 Aug 2023 15:18:18 +0100 Subject: [PATCH 24/46] Add tests for some config code --- src/HealthGPS.Console/configuration.cpp | 9 +- src/HealthGPS.Console/jsonparser.cpp | 13 + src/HealthGPS.Console/jsonparser.h | 6 + src/HealthGPS.Console/poco.h | 12 +- src/HealthGPS.Tests/CMakeLists.txt | 2 + src/HealthGPS.Tests/Configuration.Test.cpp | 327 +++++++++++++++++++++ 6 files changed, 361 insertions(+), 8 deletions(-) create mode 100644 src/HealthGPS.Tests/Configuration.Test.cpp diff --git a/src/HealthGPS.Console/configuration.cpp b/src/HealthGPS.Console/configuration.cpp index c02c440ae..a780cb13c 100644 --- a/src/HealthGPS.Console/configuration.cpp +++ b/src/HealthGPS.Console/configuration.cpp @@ -154,13 +154,8 @@ ModelInput create_model_input(core::DataTable &input_table, core::Country countr SESDefinition{.fuction_name = config.ses.function, .parameters = config.ses.parameters}; auto mapping = std::vector(); - for (auto &item : config.modelling.risk_factors) { - if (item.range.empty()) { - mapping.emplace_back(item.name, item.level); - } else { - auto boundary = hgps::OptionalRange{{item.range[0], item.range[1]}}; - mapping.emplace_back(item.name, item.level, boundary); - } + for (const auto &item : config.modelling.risk_factors) { + mapping.emplace_back(item.name, item.level, item.range); } return ModelInput(input_table, settings, run_info, ses_mapping, diff --git a/src/HealthGPS.Console/jsonparser.cpp b/src/HealthGPS.Console/jsonparser.cpp index 074403d55..2614b5aa8 100644 --- a/src/HealthGPS.Console/jsonparser.cpp +++ b/src/HealthGPS.Console/jsonparser.cpp @@ -4,6 +4,11 @@ namespace host::poco { //-------------------------------------------------------- // Risk Model JSON serialisation / de-serialisation //-------------------------------------------------------- +// Data file information +void to_json(json &j, const FileInfo &p) { + j = json{ + {"name", p.name}, {"format", p.format}, {"delimiter", p.delimiter}, {"columns", p.columns}}; +} // Linear models void to_json(json &j, const CoefficientInfo &p) { @@ -114,6 +119,14 @@ void from_json(const json &j, FactorDynamicEquationInfo &p) { j.at("ResidualsStandardDeviation").get_to(p.residuals_standard_deviation); } +// Baseline scenario adjustments +void to_json(json &j, const BaselineInfo &p) { + j = json{{"format", p.format}, + {"delimiter", p.delimiter}, + {"encoding", p.encoding}, + {"file_names", p.file_names}}; +} + // Policy Scenario void to_json(json &j, const PolicyPeriodInfo &p) { j = json{{"start_time", p.start_time}, {"finish_time", p.to_finish_time_str()}}; diff --git a/src/HealthGPS.Console/jsonparser.h b/src/HealthGPS.Console/jsonparser.h index f026ad893..b2ae9d396 100644 --- a/src/HealthGPS.Console/jsonparser.h +++ b/src/HealthGPS.Console/jsonparser.h @@ -37,6 +37,9 @@ void from_json(const json &j, HierarchicalLevelInfo &p); // Configuration sections POCO types mapping //-------------------------------------------------------- +// Data file information +void to_json(json &j, const FileInfo &p); + // Settings Information void to_json(json &j, const SettingsInfo &p); void from_json(const json &j, SettingsInfo &p); @@ -45,6 +48,9 @@ void from_json(const json &j, SettingsInfo &p); void to_json(json &j, const SESInfo &p); void from_json(const json &j, SESInfo &p); +// Baseline model information +void to_json(json &j, const BaselineInfo &p); + // Lite risk factors models (Energy Balance Model) void from_json(const json &j, RiskFactorInfo &p); diff --git a/src/HealthGPS.Console/poco.h b/src/HealthGPS.Console/poco.h index c3f21bebe..f69eaf268 100644 --- a/src/HealthGPS.Console/poco.h +++ b/src/HealthGPS.Console/poco.h @@ -14,6 +14,11 @@ struct FileInfo { std::string format; std::string delimiter; std::map columns; + + bool operator==(const FileInfo &rhs) const { + return std::tie(name, format, delimiter, columns) == + std::tie(rhs.name, rhs.format, rhs.delimiter, columns); + } }; struct SettingsInfo { @@ -32,12 +37,17 @@ struct BaselineInfo { std::string delimiter; std::string encoding; std::map file_names; + + bool operator==(const BaselineInfo &rhs) const { + return std::tie(format, delimiter, encoding, file_names) == + std::tie(rhs.format, rhs.delimiter, rhs.encoding, rhs.file_names); + } }; struct RiskFactorInfo { std::string name; int level{}; - std::vector range; + std::optional range; }; struct ModellingInfo { diff --git a/src/HealthGPS.Tests/CMakeLists.txt b/src/HealthGPS.Tests/CMakeLists.txt index decee4656..f207c6321 100644 --- a/src/HealthGPS.Tests/CMakeLists.txt +++ b/src/HealthGPS.Tests/CMakeLists.txt @@ -18,6 +18,7 @@ target_sources(HealthGPS.Tests "data_config.cpp" "AgeGenderTable.Test.cpp" "Channel.Test.cpp" + "Configuration.Test.cpp" "Core.Array2DTest.cpp" "Core.Test.cpp" "Core.UnivariateSummary.Test.cpp" @@ -46,6 +47,7 @@ target_link_libraries(HealthGPS.Tests PRIVATE HealthGPS.Core HealthGPS.Datastore + HealthGPS.LibConsole HealthGPS fmt::fmt Threads::Threads diff --git a/src/HealthGPS.Tests/Configuration.Test.cpp b/src/HealthGPS.Tests/Configuration.Test.cpp new file mode 100644 index 000000000..38e82228c --- /dev/null +++ b/src/HealthGPS.Tests/Configuration.Test.cpp @@ -0,0 +1,327 @@ +#include "pch.h" + +#include "HealthGPS.Console/configuration_parsing.h" +#include "HealthGPS.Console/configuration_parsing_helpers.h" +#include "HealthGPS.Console/jsonparser.h" + +#include +#include +#include + +using json = nlohmann::json; +using namespace host; +using namespace poco; + +#define TYPE_OF(x) std::remove_cvref_t + +namespace { +const std::string TEST_KEY = "my_key"; +const std::string TEST_KEY2 = "other_key"; + +#ifdef _WIN32 +const std::filesystem::path TEST_PATH_ABSOLUTE = R"(C:\Users\hgps_nonexistent\file.txt)"; +#else +const std::filesystem::path TEST_PATH_ABSOLUTE = "/home/hgps_nonexistent/file.txt"; +#endif + +class TempDir { + public: + TempDir() : rnd_{std::random_device()()} { + path_ = std::filesystem::path{::testing::TempDir()} / "hgps" / random_string(); + if (!std::filesystem::create_directories(path_)) { + throw std::runtime_error{"Could not create temp dir"}; + } + + path_ = std::filesystem::absolute(path_); + } + + ~TempDir() { + if (std::filesystem::exists(path_)) { + std::filesystem::remove_all(path_); + } + } + + std::string random_string() const { return std::to_string(rnd_()); } + + const auto &path() const { return path_; } + + private: + mutable std::mt19937 rnd_; + std::filesystem::path path_; + + std::filesystem::path createTempDir() { + const auto rnd = std::random_device()(); + const auto path = + std::filesystem::path{::testing::TempDir()} / "hgps" / std::to_string(rnd); + if (!std::filesystem::create_directories(path)) { + throw std::runtime_error{"Could not create temp dir"}; + } + + return std::filesystem::absolute(path); + } +}; + +class ConfigParsingFixture : public ::testing::Test { + public: + const auto &tmp_path() const { return dir_.path(); } + + std::filesystem::path random_filename() const { return dir_.random_string(); } + + std::filesystem::path create_file_relative() const { + auto file_path = random_filename(); + std::ofstream ofs{dir_.path() / file_path}; + return file_path; + } + + std::filesystem::path create_file_absolute() const { + auto file_path = tmp_path() / random_filename(); + std::ofstream ofs{file_path}; + return file_path; + } + + private: + TempDir dir_; +}; + +} // anonymous namespace + +TEST(ConfigParsing, Get) { + json j; + + // Null object + EXPECT_THROW(get(j, TEST_KEY), ConfigurationError); + + // Key missing + j[TEST_KEY2] = 1; + EXPECT_THROW(get(j, TEST_KEY), ConfigurationError); + + // Key present + j[TEST_KEY] = 2; + EXPECT_NO_THROW(get(j, TEST_KEY)); +} + +template void testGetTo(const Func &f) { + f(1); + f(std::string{"hello"}); +} + +TEST(ConfigParsing, GetToGood) { + testGetTo([](const auto &exp) { + json j; + j[TEST_KEY] = exp; + + TYPE_OF(exp) out; + EXPECT_TRUE(get_to(j, TEST_KEY, out)); + EXPECT_EQ(out, exp); + }); +} + +TEST(ConfigParsing, GetToGoodSetFlag) { + testGetTo([](const auto &exp) { + json j; + j[TEST_KEY] = exp; + + const auto check = [&exp, &j](bool initial) { + bool success = initial; + TYPE_OF(exp) out; + EXPECT_TRUE(get_to(j, TEST_KEY, out, success)); + EXPECT_EQ(success, initial); // flag shouldn't have been modified + EXPECT_EQ(out, exp); + }; + + check(true); + check(false); + }); +} + +TEST(ConfigParsing, GetToBadKey) { + testGetTo([](const auto &exp) { + json j; + j[TEST_KEY] = exp; + + // Try reading a different, non-existent key + TYPE_OF(exp) out; + EXPECT_FALSE(get_to(j, TEST_KEY2, out)); + }); +} + +TEST(ConfigParsing, GetToBadKeySetFlag) { + testGetTo([](const auto &exp) { + json j; + j[TEST_KEY] = exp; + + // Try reading a different, non-existent key + bool success = true; + TYPE_OF(exp) out; + EXPECT_FALSE(get_to(j, TEST_KEY2, out, success)); + EXPECT_FALSE(success); + }); +} + +TEST(ConfigParsing, GetToWrongType) { + testGetTo([](const auto &exp) { + json j; + j[TEST_KEY] = exp; + + // Deliberately choose a different type from the inputs so we get a mismatch + std::vector out; + EXPECT_FALSE(get_to(j, TEST_KEY, out)); + }); +} + +TEST(ConfigParsing, GetToWrongTypeSetFlag) { + testGetTo([](const auto &exp) { + json j; + j[TEST_KEY] = exp; + + // Deliberately choose a different type from the inputs so we get a mismatch + std::vector out; + bool success = true; + EXPECT_FALSE(get_to(j, TEST_KEY, out, success)); + EXPECT_FALSE(success); + }); +} + +TEST_F(ConfigParsingFixture, RebaseValidPathGood) { + { + const auto absPath = create_file_absolute(); + auto path = absPath; + ASSERT_TRUE(path.is_absolute()); + EXPECT_NO_THROW(rebase_valid_path(path, tmp_path())); + + // As path is absolute, it shouldn't be modified + EXPECT_EQ(path, absPath); + } + + { + const auto relPath = create_file_relative(); + auto path = relPath; + ASSERT_TRUE(path.is_relative()); + EXPECT_NO_THROW(rebase_valid_path(path, tmp_path())); + + // Path should have been rebased + EXPECT_EQ(path, tmp_path() / relPath); + } +} + +TEST_F(ConfigParsingFixture, RebaseValidPathBad) { + { + auto path = TEST_PATH_ABSOLUTE; + ASSERT_TRUE(path.is_absolute()); + EXPECT_THROW(rebase_valid_path(path, tmp_path()), ConfigurationError); + } + + { + auto path = random_filename(); + ASSERT_TRUE(path.is_relative()); + EXPECT_THROW(rebase_valid_path(path, tmp_path()), ConfigurationError); + } +} + +TEST_F(ConfigParsingFixture, RebaseValidPathTo) { + { + // Should fail because key doesn't exist + std::filesystem::path out; + EXPECT_FALSE(rebase_valid_path_to(json{}, TEST_KEY, out, tmp_path())); + } + + { + const auto relPath = create_file_relative(); + ASSERT_TRUE(relPath.is_relative()); + + json j; + j[TEST_KEY] = relPath; + std::filesystem::path out; + EXPECT_TRUE(rebase_valid_path_to(j, TEST_KEY, out, tmp_path())); + + // Path should have been rebased + EXPECT_EQ(out, tmp_path() / relPath); + } + + { + json j; + j[TEST_KEY] = TEST_PATH_ABSOLUTE; + + // Should fail because path is invalid + std::filesystem::path out; + EXPECT_FALSE(rebase_valid_path_to(j, TEST_KEY, out, tmp_path())); + } +} + +TEST_F(ConfigParsingFixture, RebaseValidPathToSetFlag) { + { + // Should fail because key doesn't exist + bool success = true; + std::filesystem::path out; + rebase_valid_path_to(json{}, TEST_KEY, out, tmp_path(), success); + EXPECT_FALSE(success); + } + + { + const auto relPath = create_file_relative(); + ASSERT_TRUE(relPath.is_relative()); + + json j; + j[TEST_KEY] = relPath; + bool success = true; + std::filesystem::path out; + rebase_valid_path_to(j, TEST_KEY, out, tmp_path()); + EXPECT_TRUE(success); + + // Path should have been rebased + EXPECT_EQ(out, tmp_path() / relPath); + } + + { + json j; + j[TEST_KEY] = TEST_PATH_ABSOLUTE; + auto s = j.dump(); + fmt::print(fmt::fg(fmt::color::red), "JSON: {}\n", s); + + // Should fail because path is invalid + bool success = true; + std::filesystem::path out; + rebase_valid_path_to(j, TEST_KEY, out, tmp_path(), success); + EXPECT_FALSE(success); + } +} + +TEST_F(ConfigParsingFixture, GetFileInfo) { + + const FileInfo info1{.name = create_file_absolute(), + .format = "csv", + .delimiter = ",", + .columns = {{"a", "string"}, {"b", "other string"}}}; + json j; + j["dataset"] = info1; + + /* + * Converting to JSON and back again should work. NB: We just assume that the path + * rebasing code works because it's already been tested. + */ + const auto info2 = get_file_info(j, tmp_path()); + EXPECT_EQ(info1, info2); + + // Removing a required field should cause an error + j["dataset"].erase("format"); + EXPECT_THROW(get_file_info(j, tmp_path()), ConfigurationError); +} + +TEST_F(ConfigParsingFixture, GetBaseLineInfo) { + const BaselineInfo info1{ + .format = "csv", + .delimiter = ",", + .encoding = "UTF8", + .file_names = {{"a", create_file_absolute()}, {"b", create_file_absolute()}}}; + + json j; + j["baseline_adjustments"] = info1; + + // Convert to JSON and back again + const auto info2 = get_baseline_info(j, tmp_path()); + EXPECT_EQ(info1, info2); + + // Using an invalid path should cause an error + j["baseline_adjustments"]["file_names"]["a"] = random_filename(); + EXPECT_THROW(get_baseline_info(j, tmp_path()), ConfigurationError); +} From 9f5eaecfc12d5925caaa21cb37bfc8c917e5b6bd Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Mon, 14 Aug 2023 14:36:22 +0100 Subject: [PATCH 25/46] Fix typo --- src/HealthGPS.Core/identifier.cpp | 4 ++-- src/HealthGPS.Core/identifier.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/HealthGPS.Core/identifier.cpp b/src/HealthGPS.Core/identifier.cpp index 587050d46..e0045abdf 100644 --- a/src/HealthGPS.Core/identifier.cpp +++ b/src/HealthGPS.Core/identifier.cpp @@ -12,7 +12,7 @@ Identifier Identifier::empty() { Identifier::Identifier(std::string value) : value_{to_lower(value)} { if (!value_.empty()) { - validate_identifeir(); + validate_identifier(); } hash_code_ = std::hash{}(value_); @@ -40,7 +40,7 @@ bool Identifier::equal(const Identifier &other) const noexcept { return hash_code_ == other.hash_code_; } -void Identifier::validate_identifeir() const { +void Identifier::validate_identifier() const { if (std::isdigit(value_.at(0))) { throw std::invalid_argument("Identifier must not start with a numeric value"); } diff --git a/src/HealthGPS.Core/identifier.h b/src/HealthGPS.Core/identifier.h index 63b84b4e3..0553c63ca 100644 --- a/src/HealthGPS.Core/identifier.h +++ b/src/HealthGPS.Core/identifier.h @@ -80,7 +80,7 @@ struct Identifier final { std::string value_{}; std::size_t hash_code_{std::hash{}("")}; - void validate_identifeir() const; + void validate_identifier() const; }; void from_json(const nlohmann::json &j, Identifier &id); From 6180f36ba68ce7ea810ac044e7beab67ad2519e8 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Mon, 14 Aug 2023 15:14:26 +0100 Subject: [PATCH 26/46] Allow for parsing JSON objects to any map type with Identifier as key --- src/HealthGPS.Core/identifier.cpp | 6 ------ src/HealthGPS.Core/identifier.h | 21 ++++++++++++++++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/HealthGPS.Core/identifier.cpp b/src/HealthGPS.Core/identifier.cpp index e0045abdf..668b06c31 100644 --- a/src/HealthGPS.Core/identifier.cpp +++ b/src/HealthGPS.Core/identifier.cpp @@ -58,10 +58,4 @@ std::ostream &operator<<(std::ostream &stream, const Identifier &identifier) { void from_json(const nlohmann::json &j, Identifier &id) { id = Identifier{j.get()}; } -void from_json(const nlohmann::json &j, std::map &map) { - for (auto &[key, value] : j.items()) { - map.emplace(key, value.get()); - } -} - } // namespace hgps::core diff --git a/src/HealthGPS.Core/identifier.h b/src/HealthGPS.Core/identifier.h index 0553c63ca..1a7814a9d 100644 --- a/src/HealthGPS.Core/identifier.h +++ b/src/HealthGPS.Core/identifier.h @@ -1,7 +1,9 @@ #pragma once #include +#include #include #include +#include #include "nlohmann/json.hpp" @@ -85,11 +87,28 @@ struct Identifier final { void from_json(const nlohmann::json &j, Identifier &id); -void from_json(const nlohmann::json &j, std::map &map); +namespace detail { +template